diff --git a/README.md b/README.md index 407617f4f..03377f09d 100644 --- a/README.md +++ b/README.md @@ -834,6 +834,22 @@ after a crash inherits the daemon's environment, not yours, so a forwarded secret must also be set there. `ROCM_CLI_AIM_READY_TIMEOUT_SECS` raises the 5-minute readiness wait for a cold-start model download. +`--port` is optional on the default loopback host (`127.0.0.1`): omit it and +`rocm serve` picks the first free port in `11435`-`11535`, checking both +rocm-cli's own records and the OS. The non-interactive plan output (piped +output, or the Dash approval card) shows `Port: automatic; endpoint shown +after launch` while the launch is in flight and prints the concrete endpoint +once the port is actually leased; the interactive deployment summary table +only ever shows the final resolved endpoint. Pass `--port` explicitly to pin +one; on `127.0.0.1`, an explicit loopback port already reserved by another +managed service is rejected up front, naming that service and how to free it +(`rocm services stop --yes`). A port held by an untracked process +is rejected too, but without naming the holder. `--host` to anything other +than the default +loopback address (`localhost`, `::1`, `0.0.0.0`, or any other address) +requires an explicit `--port`: automatic selection only probes `127.0.0.1` and +cannot promise a free port anywhere else. + `--gpu-memory-utilization` (a fraction above 0.0 and up to 1.0 of each GPU's total VRAM) and `--tool-call-parser` (for example `hermes`, `llama3_json`, or `mistral`, which also turns on automatic tool choice) apply to vLLM only. Lower @@ -862,6 +878,10 @@ different settings. If you previously started a service with `--temperature` flags, or with different ones, stop the existing service first (`rocm services stop --yes`) or match the original flags. +Likewise, an explicit `--port` that differs from the port the running service +is already on is refused rather than silently ignored; omit `--port` to reuse +the running service, or stop it first to serve on the new port. + By default the server runs in the background under ROCm CLI's supervision and prints a deployment summary: a progress indicator while it starts, then a table with the status, the full inference endpoint, the API-qualified model name, and a diff --git a/apps/rocm/src/chat.rs b/apps/rocm/src/chat.rs index 74fba2e49..3867d20d9 100644 --- a/apps/rocm/src/chat.rs +++ b/apps/rocm/src/chat.rs @@ -2220,6 +2220,20 @@ fn validate_chat_launch_server_tool_call(call: &providers::ChatToolCall) -> Resu if !is_loopback_host(&host) { bail!("local assistant cannot request non-local host `{host}`"); } + // A malformed `port` must be rejected here, not silently dropped by the + // argv builder and left to fall through to automatic port selection. + let port = rocm_core::launch_port_argument(object.get("port")) + .map_err(|reason| anyhow::anyhow!("ROCm tool `launch_server` argument `port` {reason}"))?; + // `rocm serve` proves a port is free only on the exact default host; every + // other spelling (including the loopback aliases `localhost`/`::1` this + // check still accepts) needs an explicit port or the CLI call refuses + // later with a less actionable error. See `is_canonical_loopback_host`. + if port.is_none() && !rocm_core::is_canonical_loopback_host(&host) { + bail!( + "local assistant must supply an explicit `port` for host `{host}`; automatic port \ + selection is only supported on the default host {DEFAULT_LOCAL_HOST}" + ); + } if object .get("device") .and_then(serde_json::Value::as_str) @@ -2719,6 +2733,16 @@ pub(crate) fn validate_chat_rocm_command_safety(args: &[String]) -> Result<()> { { bail!("local assistant cannot request non-local host `{host}`"); } + // The same rule `rocm serve` applies, so an approval card is never offered + // for a command the CLI then refuses. + let port = chat_cli_arg_value(args, "--port") + .map(crate::serve_port::parse_serve_port) + .transpose() + .map_err(anyhow::Error::msg)?; + crate::serve_port::validate_port_request( + chat_cli_arg_value(args, "--host").unwrap_or(rocm_core::DEFAULT_LOCAL_HOST), + crate::serve_port::PortRequest::from_flag(port), + )?; if chat_cli_has_flag(args, "--foreground") || !chat_cli_has_flag(args, "--managed") { bail!("local assistant must request managed serving with --managed"); } @@ -5983,6 +6007,26 @@ model recipes .to_string(); assert!(error.contains("public network binding")); + let no_port_on_custom_host = providers::ChatToolCall { + id: None, + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ + "args": ["serve", "qwen", "--host", "localhost", "--managed"] + }), + }; + let error = validate_chat_tool_call(&no_port_on_custom_host) + .unwrap_err() + .to_string(); + assert!(error.contains("explicit `--port"), "{error}"); + validate_chat_tool_call(&providers::ChatToolCall { + id: None, + name: "rocm_command".to_owned(), + arguments: serde_json::json!({ + "args": ["serve", "qwen", "--host", "localhost", "--port", "8000", "--managed"] + }), + }) + .expect("an explicit port on a loopback host is allowed"); + let foreground = providers::ChatToolCall { id: None, name: "rocm_command".to_owned(), @@ -6319,6 +6363,92 @@ model recipes assert!(error.contains("non-local host")); } + #[test] + fn chat_tool_call_requires_an_explicit_port_for_non_canonical_loopback_hosts() { + let no_port = providers::ChatToolCall { + id: None, + name: "launch_server".to_owned(), + arguments: serde_json::json!({ + "model": "tiny.gguf", + "engine": "vllm", + "host": "localhost" + }), + }; + let error = validate_chat_tool_call(&no_port).unwrap_err().to_string(); + assert!( + error.contains("explicit `port`"), + "unexpected error: {error}" + ); + + let with_port = providers::ChatToolCall { + id: None, + name: "launch_server".to_owned(), + arguments: serde_json::json!({ + "model": "tiny.gguf", + "engine": "vllm", + "host": "localhost", + "port": 11999 + }), + }; + validate_chat_tool_call(&with_port).expect("explicit port on localhost is allowed"); + + let canonical_no_port = providers::ChatToolCall { + id: None, + name: "launch_server".to_owned(), + arguments: serde_json::json!({ + "model": "tiny.gguf", + "engine": "vllm", + "host": "127.0.0.1" + }), + }; + validate_chat_tool_call(&canonical_no_port) + .expect("automatic port selection still works on the canonical loopback host"); + } + + #[test] + fn chat_tool_call_rejects_a_malformed_port_instead_of_letting_it_fall_through() { + // A non-integer `port` must not be treated as "no port supplied" here: + // the argv builder (`object.get("port").and_then(as_u64)`) would drop + // it just as silently, so without this check a malformed port on a + // non-canonical host would sail past validation only to fall through + // to automatic port selection the host doesn't support. + let string_port = providers::ChatToolCall { + id: None, + name: "launch_server".to_owned(), + arguments: serde_json::json!({ + "model": "tiny.gguf", + "engine": "vllm", + "host": "localhost", + "port": "11999" + }), + }; + let error = validate_chat_tool_call(&string_port) + .unwrap_err() + .to_string(); + assert!( + error.contains("must be an integer"), + "unexpected error: {error}" + ); + + let out_of_range_port = providers::ChatToolCall { + id: None, + name: "launch_server".to_owned(), + arguments: serde_json::json!({ + "model": "tiny.gguf", + "engine": "vllm", + "host": "127.0.0.1", + "port": 70000 + }), + }; + let error = validate_chat_tool_call(&out_of_range_port) + .unwrap_err() + .to_string(); + assert!( + error.contains("between 1 and 65535"), + "unexpected error: {error}" + ); + } + #[test] fn chat_tool_call_rejects_bad_install_suggestions() { for (arguments, expected) in [ diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 15fe175ca..964bb8bc8 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -39,6 +39,7 @@ mod runtime_install_finalize; mod runtime_services; mod runtimes_cmd; mod serve_cmd; +mod serve_port; mod serve_summary; mod service_hints; mod service_launch; @@ -531,9 +532,16 @@ rocm serve qwen --verbose --device gpu_required")] /// Host address to bind. #[arg(long, default_value = DEFAULT_LOCAL_HOST)] host: String, - /// TCP port to bind. - #[arg(long, default_value_t = rocm_core::DEFAULT_LOCAL_PORT)] - port: u16, + /// TCP port to bind. Omit it to choose an available port automatically, + /// starting at 11435, on the default 127.0.0.1 host. Any other --host + /// requires an explicit --port: proactive collision detection is claimed + /// only for local 127.0.0.1 serving. + // `allow_hyphen_values` so a negative value such as `-1` reaches + // `parse_serve_port` and gets the domain error naming the valid range, + // instead of clap rejecting it as an unknown flag. Mirrors the same + // choice made for `--gpu-memory-utilization` below. + #[arg(long, value_parser = serve_port::parse_serve_port, allow_hyphen_values = true)] + port: Option, /// Attach to the server in this terminal and stream its logs (Ctrl-D to /// detach and leave it running, Ctrl-C to stop). Same as --verbose. #[arg(long, conflicts_with = "managed")] @@ -4565,8 +4573,11 @@ fn apply_service_prune_plan( /// window — [`select_gpu_indices_under_launch_lock`] acquires it, hands the /// guard back to its caller, and `start_managed_service`/`run_attached_service` /// only drop it once the record is persisted (see that function's own comment, -/// which this one must not drift from). The gap was simply that prune never -/// acquired it: the helper above was the single acquirer in the tree. Taking it +/// which this one must not drift from). `restart_internal_managed_service_with` +/// acquires the same lock around its own respawn, for the same reason. The gap +/// was simply that prune never acquired it, while those two already did +/// (`rocmd`'s recovery path takes the same file lock too, bounded, from its own +/// process — see `restart_managed_service`'s doc comment). Taking it /// here closes the window deterministically, for any number of records, which no /// age floor can — the window is not short and is not bounded, because /// `spawn_managed_engine_child`'s idempotency guard runs `load_managed_services` @@ -4624,9 +4635,9 @@ fn apply_service_prune_plan( /// **No nested acquire.** `FileLock::acquire` blocks with no `try_` variant, so /// a second acquire on this same lock path while this process already holds /// it would hang forever. Nothing reachable from either phase re-acquires -/// `managed_launch_lock_path`: the only other production call site on this -/// same path is `select_gpu_indices_under_launch_lock`, which is only ever -/// reached from `serve`, and `serve` never runs prune in-process or as a +/// `managed_launch_lock_path`: the other production acquirers of this +/// same path are `select_gpu_indices_under_launch_lock` (reached only from +/// `serve`) and `rocm services restart`, and neither runs prune in-process or as a /// subprocess (test code acquires this path directly too, but that is test /// setup, not something reachable from this function's own call graph). /// @@ -5932,6 +5943,21 @@ pub(crate) mod tests { .collect() } + fn serve_arg_help(arg_id: &str) -> String { + let mut cli = Cli::command(); + let serve = cli + .find_subcommand_mut("serve") + .expect("serve subcommand exists"); + let arg = serve + .get_arguments() + .find(|arg| arg.get_id().as_str() == arg_id) + .unwrap_or_else(|| panic!("serve has no `{arg_id}` argument")); + arg.get_long_help() + .map(ToString::to_string) + .or_else(|| arg.get_help().map(ToString::to_string)) + .unwrap_or_default() + } + // Both `--engine` and `--device` restrict their input to a fixed set via a // clap `value_parser`/`ValueEnum`, so invalid input is rejected as a usage // error (exit code 2) with the accepted choices listed. Values are advertised @@ -6114,6 +6140,77 @@ pub(crate) mod tests { } } + #[test] + fn serve_without_port_flag_requests_automatic_selection() { + let cli = parse_serve(&[]).expect("bare serve parses"); + match cli.command { + Some(Command::Serve { port, host, .. }) => { + assert_eq!(port, None, "omitted --port must stay unresolved"); + assert_eq!(host, DEFAULT_LOCAL_HOST); + assert!(serve_port::PortRequest::from_flag(port).is_auto()); + } + other => panic!("expected Serve, got {other:?}"), + } + } + + #[test] + fn serve_with_explicit_port_keeps_it_concrete() { + let cli = parse_serve(&["--port", "8000"]).expect("explicit port parses"); + match cli.command { + Some(Command::Serve { port, .. }) => { + assert_eq!(port, Some(8000)); + assert_eq!( + serve_port::PortRequest::from_flag(port), + serve_port::PortRequest::Explicit(8000) + ); + } + other => panic!("expected Serve, got {other:?}"), + } + } + + #[test] + fn serve_rejects_a_zero_or_out_of_range_port() { + for bad in ["0", "65536", "-1", "auto"] { + let error = parse_serve(&["--port", bad]).expect_err("must be rejected"); + assert!( + error.to_string().contains("between 1 and 65535"), + "unexpected error for --port {bad}: {error}" + ); + } + } + + #[test] + fn hidden_engine_serve_command_keeps_a_concrete_default_port() { + // Only the user-facing verb gained automatic selection; the direct engine + // entry point still requires (and defaults to) a concrete port. + let cli = Cli::try_parse_from([ + "rocm", + "__engine-serve-http", + "lemonade", + "svc-1", + "qwen", + "--state-path", + "state.json", + ]) + .expect("hidden engine command parses"); + match cli.command { + Some(Command::EngineServeHttp { port, host, .. }) => { + assert_eq!(port, rocm_core::DEFAULT_LOCAL_PORT); + assert_eq!(host, DEFAULT_LOCAL_HOST); + } + other => panic!("expected EngineServeHttp, got {other:?}"), + } + } + + #[test] + fn serve_port_help_documents_omission_and_the_custom_host_rule() { + let help = serve_arg_help("port"); + assert!(help.contains("11435"), "{help}"); + assert!(help.contains("automatically"), "{help}"); + assert!(help.contains("127.0.0.1"), "{help}"); + assert!(help.contains("explicit --port"), "{help}"); + } + #[test] fn serve_verbose_conflicts_with_managed() { // `--verbose` streams logs in the foreground; a backgrounded managed diff --git a/apps/rocm/src/managed_service_status.rs b/apps/rocm/src/managed_service_status.rs index 99b71a2ba..5fffe9b8c 100644 --- a/apps/rocm/src/managed_service_status.rs +++ b/apps/rocm/src/managed_service_status.rs @@ -14,7 +14,7 @@ //! registry (`load_managed_services`, `refresh_managed_service_runtime_liveness`, //! `recorded_service_pids`, `settle_pending_stop_key_cleanup`), the idempotency //! guard that finds an already-live service for a given engine+model -//! (`existing_live_managed_service`), the "is this record live" classifier +//! (`existing_live_managed_service_in`), the "is this record live" classifier //! (`managed_service_is_live`), and the dashboard sidebar's ready/starting/past- //! attempts counts and one-line status text //! (`ManagedServiceSidebarCounts`/`managed_service_sidebar_counts`/ @@ -74,15 +74,13 @@ pub(crate) fn managed_service_sidebar_counts( } pub(crate) fn managed_service_is_live(record: &ManagedServiceRecord) -> bool { - matches!( - record.status.as_str(), - "ready" | "running" | "starting" | "recovering" - ) + record.is_live() } /// Idempotency guard for managed launches: returns an already-live managed -/// service for this engine+model, if any. The caller compares its effective -/// launch recipe before deciding whether reuse is safe. +/// service for this engine+model, if any, from an already-refreshed live set. +/// The caller compares its effective launch recipe before deciding whether reuse +/// is safe. /// /// Keyed on `(engine, canonical_model_id)` — NOT `service_id`. `generate_service_id` /// embeds `unix_time_millis()`, so every launch mints a unique id; matching on it @@ -92,19 +90,17 @@ pub(crate) fn managed_service_is_live(record: &ManagedServiceRecord) -> bool { /// the newest live match. Prevents a second `serve --managed` for the same /// engine+model from spawning a duplicate process once the TUI job-bridge guard /// has cleared. -pub(crate) fn existing_live_managed_service( - paths: &AppPaths, +/// +/// Takes the snapshot rather than the paths so the allocation transaction loads +/// it once and uses it for both this check and the port reservations: duplicate +/// detection and candidate selection can never observe different states. +pub(crate) fn existing_live_managed_service_in<'a>( + live: &'a [ManagedServiceRecord], engine: &str, canonical_model_id: &str, -) -> Option { - load_managed_services(paths) - .ok()? - .into_iter() - .find(|record| { - record.engine == engine - && record.canonical_model_id == canonical_model_id - && managed_service_is_live(record) - }) +) -> Option<&'a ManagedServiceRecord> { + live.iter() + .find(|record| record.engine == engine && record.canonical_model_id == canonical_model_id) } const SERVICE_LIVENESS_CHECK_TIMEOUT: Duration = Duration::from_millis(750); @@ -501,145 +497,6 @@ mod tests { Ok(()) } - #[test] - fn duplicate_managed_launch_detected_across_distinct_service_ids() -> Result<()> { - // `generate_service_id` embeds a timestamp, so a second launch for the - // same engine+model has a DIFFERENT service_id. The guard must still - // detect the live service by (engine, canonical_model_id), and return - // the newest live match. `starting` skips the endpoint probe; the - // current process id is a guaranteed-live PID. - let (root, paths) = test_paths("dup-managed-distinct-ids"); - paths.ensure()?; - - // Older, dead manifest for the same engine+model (distinct service_id). - let mut dead = ManagedServiceRecord::new( - &paths, - "lemonade-qwen-1000", - "lemonade", - "qwen", - "qwen-canonical", - "127.0.0.1", - 11500, - "managed", - 999_999_999, - None, - None, - None, - ); - dead.status = "ready".to_owned(); - dead.engine_pid = Some(999_999_999); - dead.created_at_unix_ms = 1000; - dead.write()?; - - // Newer, live manifest for the same engine+model (distinct service_id). - let mut live = ManagedServiceRecord::new( - &paths, - "lemonade-qwen-2000", - "lemonade", - "qwen", - "qwen-canonical", - "127.0.0.1", - 11501, - "managed", - std::process::id(), - None, - None, - None, - ); - live.status = "starting".to_owned(); - live.engine_pid = Some(std::process::id()); - live.created_at_unix_ms = 2000; - live.write()?; - - let found = existing_live_managed_service(&paths, "lemonade", "qwen-canonical"); - let _ = fs::remove_dir_all(root); - - let found = found.expect("a live managed service should be detected by engine+model"); - assert_eq!( - found.service_id, "lemonade-qwen-2000", - "should return the newest live match" - ); - assert!(managed_service_is_live(&found)); - Ok(()) - } - - #[test] - fn dead_managed_service_allows_relaunch() -> Result<()> { - // A stale manifest with dead PIDs must NOT block a relaunch: liveness - // refresh demotes it to "stopped", so the guard returns None. - let (root, paths) = test_paths("dup-managed-dead"); - paths.ensure()?; - let mut record = ManagedServiceRecord::new( - &paths, - "lemonade-qwen-3000", - "lemonade", - "qwen", - "qwen-canonical", - "127.0.0.1", - 11502, - "managed", - 999_999_999, - None, - None, - None, - ); - record.status = "ready".to_owned(); - record.engine_pid = Some(999_999_999); - record.write()?; - - let found = existing_live_managed_service(&paths, "lemonade", "qwen-canonical"); - let _ = fs::remove_dir_all(root); - - assert!( - found.is_none(), - "a dead managed service must not block relaunch" - ); - Ok(()) - } - - #[test] - fn live_service_for_other_model_does_not_block() -> Result<()> { - // A live service for a DIFFERENT model must not match — the guard keys - // on the model, not just the engine. - let (root, paths) = test_paths("dup-managed-other-model"); - paths.ensure()?; - let mut record = ManagedServiceRecord::new( - &paths, - "lemonade-other-1", - "lemonade", - "other", - "other-canonical", - "127.0.0.1", - 11503, - "managed", - std::process::id(), - None, - None, - None, - ); - record.status = "starting".to_owned(); - record.engine_pid = Some(std::process::id()); - record.write()?; - - let found = existing_live_managed_service(&paths, "lemonade", "qwen-canonical"); - let _ = fs::remove_dir_all(root); - - assert!( - found.is_none(), - "a live service for a different model must not match" - ); - Ok(()) - } - - #[test] - fn missing_manifest_allows_launch() { - // No services dir / manifests → nothing to detect, launch proceeds. - let (root, paths) = test_paths("dup-managed-missing"); - let found = existing_live_managed_service(&paths, "lemonade", "qwen-canonical"); - let _ = fs::remove_dir_all(root); - assert!(found.is_none()); - } - /// A public record with a dead pid and a stored endpoint key, for the /// liveness-refresh cases below. The port is one nothing listens on, so the /// refresh's endpoint probe fails and it falls through to the pid check. diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 22f02bd0e..0f2f46101 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -51,6 +51,7 @@ use crate::gpu_policy::{ detect_gpu_count, gpu_vram_usage, parse_device_policy, parse_gpu_selection, select_gpu_indices_under_launch_lock, serve_gpu_low_memory_warning, validate_pinned_gpu_index, }; +use crate::serve_port; use crate::serve_summary; use crate::service_launch::{ drop_orphaned_endpoint_key_on_already_running, ensure_background_helper_running_quiet, @@ -567,7 +568,7 @@ pub(crate) struct ServeArgs { pub(crate) runtime_id: Option, pub(crate) env_id: Option, pub(crate) host: String, - pub(crate) port: u16, + pub(crate) port: Option, pub(crate) foreground: bool, pub(crate) managed: bool, pub(crate) verbose: bool, @@ -607,8 +608,10 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { api_key, engine_env, } = args; + let port_request = serve_port::PortRequest::from_flag(port); let _ = managed; // background is now the default; --managed is accepted as an explicit synonym. validate_bind_host(&host, allow_public_bind)?; + serve_port::validate_port_request(&host, port_request)?; // Loopback stays credential-free; a public bind must be authenticated. Resolve // (or generate) the endpoint key now so every downstream path — engine spawn, // readiness probe, smoke test, and the client-config we print — shares one value. @@ -850,7 +853,10 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { println!(" engine: {selected_engine}"); println!("{}", serve_engine_selection_line(&serve_engine)); println!(" host: {host}"); - println!(" port: {port}"); + match port { + Some(port) => println!(" port: {port}"), + None => println!(" {}", serve_port::AUTOMATIC_PORT_DISCLOSURE), + } if let Some(runtime_id) = resolved_selection.runtime_id.as_deref() { println!(" runtime_id: {runtime_id}"); } @@ -931,12 +937,13 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { cli_progress::Spinner::new(format!("Starting {model} on {selected_engine}…")); spinner.tick(); let report = start_managed_service( + &paths, &selected_engine, &service_id, &model, &resolve, &host, - port, + port_request, &resolve.device_policy, &gpu_indices, managed_runtime_id.as_deref(), @@ -1020,12 +1027,13 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { } run_attached_service( + &paths, &selected_engine, &service_id, &model, &resolve, &host, - port, + port_request, &gpu_indices, resolved_selection.runtime_id.as_deref(), resolved_selection.env_id.as_deref(), diff --git a/apps/rocm/src/serve_port.rs b/apps/rocm/src/serve_port.rs new file mode 100644 index 000000000..7bc8a638b --- /dev/null +++ b/apps/rocm/src/serve_port.rs @@ -0,0 +1,547 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Central serve-port policy for the user-facing `rocm serve` verb. +//! +//! `--port` is an explicit-or-automatic *request*, not a concrete default. This +//! module turns that request into one concrete, leased port before any manifest, +//! log, endpoint key, or child process exists: +//! +//! - **Automatic** (`--port` omitted) is accepted only on the canonical default +//! loopback host `127.0.0.1`, and scans [`AUTO_PORT_FIRST`]..=[`AUTO_PORT_LAST`] +//! for a candidate that is neither reserved by a live managed service nor +//! occupied on the OS. +//! - **Explicit on canonical loopback** goes through the same reservation and +//! OS preflight, so a collision fails immediately with a clear message instead +//! of surfacing as an engine bind error minutes into startup. +//! - **Explicit on a custom host** (`localhost`, `::1`, `0.0.0.0`, any other +//! IPv4/IPv6 address) is passed through untouched: bind success or failure +//! stays engine-owned, because an IPv4-loopback probe proves nothing about +//! those addresses. Automatic is refused there rather than guessing. +//! +//! Everything downstream of this module — engine requests, service records, +//! readiness, supervision, recovery — keeps its concrete `u16` contract. + +use crate::service_hints; +use anyhow::{Result, bail}; +use rocm_core::{DEFAULT_LOCAL_HOST, DEFAULT_LOCAL_PORT, LoopbackPortLease}; +use std::io; + +/// First automatic candidate. Also the legacy well-known local endpoint, so an +/// otherwise-idle machine keeps serving on the port users already know. +pub(crate) const AUTO_PORT_FIRST: u16 = DEFAULT_LOCAL_PORT; +/// Number of candidates *after* the first one. The scanned range is inclusive on +/// both ends: `11435..=11535`, 101 candidates. +pub(crate) const AUTO_PORT_SPAN: u16 = 100; +/// Last automatic candidate. +pub(crate) const AUTO_PORT_LAST: u16 = AUTO_PORT_FIRST + AUTO_PORT_SPAN; + +/// The one phrase used wherever a port must be described *before* it is +/// resolved. Re-exported from the Dash approval card's own constant so the CLI +/// plan output and the wizard can never drift apart in wording. +pub(crate) use rocm_dash_tui::ui::serve_wizard::AUTO_PORT_NOTE as AUTOMATIC_PORT_DISCLOSURE; + +/// What the user asked for on the command line. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum PortRequest { + /// `--port` omitted: choose an available local port. + Auto, + /// `--port N`: use exactly this port. + Explicit(u16), +} + +impl PortRequest { + #[must_use] + pub(crate) const fn from_flag(port: Option) -> Self { + match port { + Some(port) => Self::Explicit(port), + None => Self::Auto, + } + } + + #[must_use] + pub(crate) const fn is_auto(self) -> bool { + matches!(self, Self::Auto) + } + + /// The concrete port, when the user named one. + #[must_use] + pub(crate) const fn explicit(self) -> Option { + match self { + Self::Auto => None, + Self::Explicit(port) => Some(port), + } + } +} + +/// A port a live managed service still owns. Carried with its service id so a +/// collision names the thing actually holding the port. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct ReservedPort { + pub(crate) port: u16, + pub(crate) service_id: String, + pub(crate) status: String, +} + +/// clap value parser for every user-facing `--port`. +/// +/// Rejects `0` explicitly: the OS reads it as "any ephemeral port", which would +/// silently produce an endpoint nobody asked for. +/// +/// # Errors +/// Returns a user-facing message for anything outside 1-65535. +pub(crate) fn parse_serve_port(value: &str) -> Result { + match value.trim().parse::() { + Ok(0) | Err(_) => Err(format!( + "port must be a number between 1 and 65535, got '{value}'" + )), + Ok(port) => Ok(port), + } +} + +/// Reject an automatic request on a host this policy cannot preflight, before +/// any engine resolution or side effect happens. +/// +/// # Errors +/// Fails when `--port` was omitted for anything but the canonical loopback host. +pub(crate) fn validate_port_request(host: &str, request: PortRequest) -> Result<()> { + if request.is_auto() && !rocm_core::is_canonical_loopback_host(host) { + bail!( + "`rocm serve --host {host}` needs an explicit `--port <1-65535>`. Automatic port \ + selection is supported only on the default host {DEFAULT_LOCAL_HOST}, where ROCm \ + can prove a port is free before launching; on any other address the engine owns \ + the bind." + ); + } + Ok(()) +} + +/// Production bind probe: take a real `127.0.0.1:` lease. +/// +/// # Errors +/// Propagates the OS bind error verbatim so the caller can distinguish +/// "occupied" from "not permitted"/"not available". +pub(crate) fn loopback_bind_probe(port: u16) -> io::Result { + rocm_core::lease_loopback_port(port) +} + +/// Turn a [`PortRequest`] into one concrete, leased port. +/// +/// Callers must hold the shared managed-service allocation lock across this call +/// *and* the record publication that follows it, so a concurrent launch cannot +/// pick the same candidate between the probe and the spawn. +/// +/// `probe` is injected so tests can drive deterministic OS outcomes; production +/// passes [`loopback_bind_probe`]. +/// +/// # Errors +/// Fails when an explicit port is reserved or occupied, when the automatic range +/// is exhausted, or when the OS reports anything other than "address in use" — +/// a permission or address-availability error must never be papered over by +/// quietly choosing a different port. +pub(crate) fn resolve_serve_port( + host: &str, + request: PortRequest, + reserved: &[ReservedPort], + probe: &mut dyn FnMut(u16) -> io::Result, +) -> Result { + if !rocm_core::is_canonical_loopback_host(host) { + // Custom host: an explicit port passes through untouched so bind success + // or failure stays engine-owned. Automatic is refused here as well as in + // `serve()`, because this is the shared choke point every caller reaches + // — and it is refused by returning an error, never by panicking. + validate_port_request(host, request)?; + let Some(port) = request.explicit() else { + bail!("automatic port selection is not available on host {host}"); + }; + return Ok(LoopbackPortLease::engine_owned(port)); + } + + match request { + PortRequest::Explicit(port) => { + if let Some(owner) = reserved.iter().find(|entry| entry.port == port) { + // The "stop it" advice below is proven, not just worded: see + // `stopping_the_reserving_record_frees_its_explicit_port_for_reuse` + // in `service_launch.rs`, which recreates the exact on-disk status + // transition `rocm services stop` performs and confirms the same + // explicit port then resolves. A full CLI-level E2E for this needs + // a real GPU, since reaching this branch requires a completed + // model/engine resolution first. + bail!( + "{DEFAULT_LOCAL_HOST}:{port} is already reserved by managed service {} \ + ({}). Stop it with `{}`, or pass a different `--port`.", + owner.service_id, + owner.status, + service_hints::service_stop_hint(&owner.service_id) + ); + } + probe(port).map_err(|error| explicit_bind_error(port, &error)) + } + PortRequest::Auto => { + for port in AUTO_PORT_FIRST..=AUTO_PORT_LAST { + if reserved.iter().any(|entry| entry.port == port) { + continue; + } + match probe(port) { + Ok(lease) => return Ok(lease), + // Occupied is the only outcome that advances the scan. On + // Windows, a port inside an excluded range (Hyper-V, WSL, + // Docker) or held with exclusive-address-use fails with + // WSAEACCES, which Rust reports as `PermissionDenied`, not + // `AddrInUse` — treat it the same as occupied there. + Err(error) if error.kind() == io::ErrorKind::AddrInUse => {} + #[cfg(windows)] + Err(error) if error.kind() == io::ErrorKind::PermissionDenied => {} + Err(error) => bail!( + "failed to reserve {DEFAULT_LOCAL_HOST}:{port} while choosing an \ + automatic port: {error}" + ), + } + } + bail!( + "no free TCP port for {DEFAULT_LOCAL_HOST} in {AUTO_PORT_FIRST}-{AUTO_PORT_LAST}; \ + every candidate is reserved by a live managed service or already in use. Stop an \ + unused server with `{}`, or pass an explicit `--port`.", + service_hints::service_action_hint("stop", "") + ); + } + } +} + +fn explicit_bind_error(port: u16, error: &io::Error) -> anyhow::Error { + if error.kind() == io::ErrorKind::AddrInUse { + return anyhow::anyhow!( + "{DEFAULT_LOCAL_HOST}:{port} is already in use by another process. Choose a free \ + `--port`, stop the process holding it, or omit `--port` to let ROCm pick an \ + available port starting at {AUTO_PORT_FIRST}." + ); + } + anyhow::anyhow!("failed to reserve {DEFAULT_LOCAL_HOST}:{port}: {error}") +} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::TcpListener; + + fn reserved(port: u16, service_id: &str, status: &str) -> ReservedPort { + ReservedPort { + port, + service_id: service_id.to_owned(), + status: status.to_owned(), + } + } + + /// A probe driven by a table of scripted per-port failures. Any port absent + /// from the table binds successfully. + fn scripted( + outcomes: Vec<(u16, io::ErrorKind)>, + ) -> impl FnMut(u16) -> io::Result { + move |port| match outcomes.iter().find(|(candidate, _)| *candidate == port) { + Some((_, kind)) => Err(io::Error::new(*kind, "scripted")), + None => Ok(LoopbackPortLease::engine_owned(port)), + } + } + + #[test] + fn port_request_distinguishes_omitted_from_explicit() { + assert_eq!(PortRequest::from_flag(None), PortRequest::Auto); + assert!(PortRequest::from_flag(None).is_auto()); + assert_eq!(PortRequest::from_flag(None).explicit(), None); + assert_eq!( + PortRequest::from_flag(Some(8000)), + PortRequest::Explicit(8000) + ); + assert!(!PortRequest::from_flag(Some(8000)).is_auto()); + assert_eq!(PortRequest::from_flag(Some(8000)).explicit(), Some(8000)); + } + + #[test] + fn port_parser_rejects_zero_and_out_of_range_values() { + assert_eq!(parse_serve_port("8000"), Ok(8000)); + assert_eq!(parse_serve_port(" 11435 "), Ok(11_435)); + assert_eq!(parse_serve_port("65535"), Ok(65_535)); + for bad in ["0", "65536", "-1", "eleven", ""] { + let error = parse_serve_port(bad).expect_err("must be rejected"); + assert!( + error.contains("between 1 and 65535"), + "unexpected message for {bad:?}: {error}" + ); + } + } + + #[test] + fn automatic_range_is_exactly_the_documented_window() { + assert_eq!(AUTO_PORT_FIRST, 11_435); + assert_eq!(AUTO_PORT_LAST, 11_535); + assert_eq!((AUTO_PORT_FIRST..=AUTO_PORT_LAST).count(), 101); + } + + #[test] + fn automatic_is_refused_on_every_custom_host() { + validate_port_request(DEFAULT_LOCAL_HOST, PortRequest::Auto).expect("canonical host is ok"); + for host in ["localhost", "::1", "0.0.0.0", "127.0.0.2", "10.0.0.5"] { + let error = validate_port_request(host, PortRequest::Auto) + .expect_err("automatic must be refused on a custom host"); + let message = format!("{error:#}"); + assert!(message.contains("explicit `--port"), "{message}"); + assert!(message.contains(host), "{message}"); + // An explicit port is always accepted there. + validate_port_request(host, PortRequest::Explicit(8000)) + .expect("explicit port is valid on a custom host"); + } + } + + #[test] + fn custom_host_explicit_port_bypasses_the_loopback_preflight() { + let mut probe = |_port: u16| -> io::Result { + panic!("custom hosts must not be preflighted against IPv4 loopback") + }; + // Even a port a live loopback service reserves is passed straight + // through: the engine owns bind on that address. + let lease = resolve_serve_port( + "0.0.0.0", + PortRequest::Explicit(11_435), + &[reserved(11_435, "svc-a", "running")], + &mut probe, + ) + .expect("custom host explicit port is engine-owned"); + assert_eq!(lease.port(), 11_435); + assert!(!lease.is_held()); + } + + #[test] + fn resolving_auto_on_a_custom_host_errors_instead_of_panicking() { + // The choke point must refuse Auto on its own, even when a caller skipped + // the earlier `validate_port_request` gate — as a Result, never a panic. + let mut probe = |_port: u16| -> io::Result { + panic!("a custom host must never be preflighted") + }; + for host in ["localhost", "::1", "0.0.0.0", "10.0.0.5"] { + let error = resolve_serve_port(host, PortRequest::Auto, &[], &mut probe) + .expect_err("automatic selection is not supported on a custom host"); + let message = format!("{error:#}"); + assert!(message.contains("explicit `--port"), "{message}"); + assert!(message.contains(host), "{message}"); + } + } + + /// Dash accepts exactly the hosts the CLI treats as non-public, so a form + /// Dash approves is never refused by the CLI for lacking + /// `--allow-public-bind`. + #[test] + fn dash_loopback_hosts_match_the_clis_non_public_hosts() { + use rocm_dash_tui::ui::serve_wizard::{LOOPBACK_HOSTS, is_loopback_host}; + for host in LOOPBACK_HOSTS { + assert!(!rocm_engine_protocol::is_public_bind_host(host), "{host}"); + } + for host in ["LOCALHOST", "[::1]", "127.0.0.2", "0.0.0.0", "::"] { + assert_eq!( + is_loopback_host(host), + !rocm_engine_protocol::is_public_bind_host(host), + "{host}" + ); + } + } + + #[test] + fn explicit_loopback_port_outside_the_auto_range_is_retained() { + let mut probe = scripted(vec![]); + let lease = resolve_serve_port( + DEFAULT_LOCAL_HOST, + PortRequest::Explicit(8000), + &[], + &mut probe, + ) + .expect("a free explicit port is kept"); + assert_eq!(lease.port(), 8000); + } + + #[test] + fn explicit_loopback_port_that_is_occupied_is_rejected() { + let mut probe = scripted(vec![(8000, io::ErrorKind::AddrInUse)]); + let error = resolve_serve_port( + DEFAULT_LOCAL_HOST, + PortRequest::Explicit(8000), + &[], + &mut probe, + ) + .expect_err("an occupied explicit port must fail"); + let message = format!("{error:#}"); + assert!(message.contains("127.0.0.1:8000"), "{message}"); + assert!(message.contains("already in use"), "{message}"); + } + + #[test] + fn explicit_loopback_port_reserved_by_a_live_service_names_that_service() { + let mut probe = |_port: u16| -> io::Result { + panic!("a reserved port must be rejected before probing") + }; + let error = resolve_serve_port( + DEFAULT_LOCAL_HOST, + PortRequest::Explicit(11_435), + &[reserved(11_435, "vllm-qwen-1", "starting")], + &mut probe, + ) + .expect_err("a reserved explicit port must fail"); + let message = format!("{error:#}"); + assert!(message.contains("vllm-qwen-1"), "{message}"); + assert!(message.contains("starting"), "{message}"); + } + + #[test] + fn automatic_takes_the_first_candidate_when_it_is_free() { + let mut probe = scripted(vec![]); + let lease = resolve_serve_port(DEFAULT_LOCAL_HOST, PortRequest::Auto, &[], &mut probe) + .expect("first candidate is free"); + assert_eq!(lease.port(), AUTO_PORT_FIRST); + } + + #[test] + fn automatic_skips_occupied_candidates_until_one_binds() { + let mut probe = scripted(vec![ + (11_435, io::ErrorKind::AddrInUse), + (11_436, io::ErrorKind::AddrInUse), + (11_437, io::ErrorKind::AddrInUse), + ]); + let lease = resolve_serve_port(DEFAULT_LOCAL_HOST, PortRequest::Auto, &[], &mut probe) + .expect("scan advances past occupied candidates"); + assert_eq!(lease.port(), 11_438); + } + + #[test] + fn automatic_skips_ports_reserved_by_live_records_without_probing_them() { + let mut probed: Vec = Vec::new(); + let lease = { + let mut probe = |port: u16| -> io::Result { + probed.push(port); + Ok(LoopbackPortLease::engine_owned(port)) + }; + resolve_serve_port( + DEFAULT_LOCAL_HOST, + PortRequest::Auto, + &[ + reserved(11_435, "svc-ready", "ready"), + reserved(11_436, "svc-recovering", "recovering"), + ], + &mut probe, + ) + .expect("scan skips reserved candidates") + }; + assert_eq!(lease.port(), 11_437); + assert_eq!(probed, vec![11_437]); + } + + #[test] + fn automatic_stops_immediately_on_a_non_collision_bind_error() { + // `PermissionDenied` is excluded here: on Windows it is how a reserved + // range or an exclusive-address-use holder reports occupied, so it + // advances the scan there (see the Windows-only test below). On every + // other platform it means what it says, and the other-platform half + // of that behavior is covered by + // `permission_denied_does_not_advance_the_scan_off_windows`. + let mut probe = scripted(vec![(11_435, io::ErrorKind::AddrNotAvailable)]); + let error = resolve_serve_port(DEFAULT_LOCAL_HOST, PortRequest::Auto, &[], &mut probe) + .expect_err("only AddrInUse (or, on Windows, PermissionDenied) may advance the scan"); + let message = format!("{error:#}"); + assert!(message.contains("127.0.0.1:11435"), "{message}"); + } + + #[test] + #[cfg(not(windows))] + fn permission_denied_does_not_advance_the_scan_off_windows() { + let mut probe = scripted(vec![(11_435, io::ErrorKind::PermissionDenied)]); + let error = resolve_serve_port(DEFAULT_LOCAL_HOST, PortRequest::Auto, &[], &mut probe) + .expect_err("a permission error must not be papered over by choosing another port"); + let message = format!("{error:#}"); + assert!(message.contains("127.0.0.1:11435"), "{message}"); + } + + #[test] + #[cfg(windows)] + fn permission_denied_advances_the_scan_on_windows() { + // WSAEACCES (reserved range or exclusive-address-use) surfaces as + // `PermissionDenied`, not `AddrInUse`; one such candidate must not + // abort automatic selection on a platform #239 asks to support. + let mut probe = scripted(vec![(11_435, io::ErrorKind::PermissionDenied)]); + let lease = resolve_serve_port(DEFAULT_LOCAL_HOST, PortRequest::Auto, &[], &mut probe) + .expect("the scan advances past a Windows-reserved candidate"); + assert_eq!(lease.port(), 11_436); + } + + #[test] + fn explicit_non_collision_bind_error_reports_host_port_and_os_detail() { + let mut probe = |_port: u16| -> io::Result { + Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "permission denied", + )) + }; + let error = resolve_serve_port( + DEFAULT_LOCAL_HOST, + PortRequest::Explicit(80), + &[], + &mut probe, + ) + .expect_err("a privileged port must fail"); + let message = format!("{error:#}"); + assert!(message.contains("127.0.0.1:80"), "{message}"); + assert!(message.contains("permission denied"), "{message}"); + } + + #[test] + fn automatic_exhaustion_reports_the_exact_scanned_range() { + let probe = |_port: u16| -> io::Result { + Err(io::Error::new(io::ErrorKind::AddrInUse, "occupied")) + }; + let mut probes = 0usize; + let mut counting = |port: u16| { + probes += 1; + probe(port) + }; + let error = resolve_serve_port(DEFAULT_LOCAL_HOST, PortRequest::Auto, &[], &mut counting) + .expect_err("an exhausted range must fail"); + let message = format!("{error:#}"); + assert!(message.contains("11435-11535"), "{message}"); + assert_eq!(probes, 101, "every port in the inclusive window is probed"); + } + + #[test] + fn automatic_reaches_the_last_port_of_the_window() { + let occupied = (AUTO_PORT_FIRST..AUTO_PORT_LAST) + .map(|port| (port, io::ErrorKind::AddrInUse)) + .collect(); + let mut probe = scripted(occupied); + let lease = resolve_serve_port(DEFAULT_LOCAL_HOST, PortRequest::Auto, &[], &mut probe) + .expect("the final port of the window is still tried"); + assert_eq!(lease.port(), AUTO_PORT_LAST); + } + + #[test] + fn automatic_selects_against_real_loopback_listeners() { + // Real sockets, not a script: two live listeners hold the first two + // candidates and the production probe must walk past them. The + // candidates are remapped onto ephemeral ports so a real `rocm serve` + // sitting in 11435-11535 on this machine can neither fail nor + // vacuously skip the test. + let first = TcpListener::bind(("127.0.0.1", 0)).expect("bind"); + let second = TcpListener::bind(("127.0.0.1", 0)).expect("bind"); + let free = TcpListener::bind(("127.0.0.1", 0)).expect("bind"); + let targets = [ + first.local_addr().expect("addr").port(), + second.local_addr().expect("addr").port(), + free.local_addr().expect("addr").port(), + ]; + drop(free); + let mut probe = |candidate: u16| { + let index = usize::from(candidate - AUTO_PORT_FIRST); + loopback_bind_probe(targets[index.min(targets.len() - 1)]) + }; + let lease = resolve_serve_port(DEFAULT_LOCAL_HOST, PortRequest::Auto, &[], &mut probe) + .expect("the third candidate is free"); + assert_eq!(lease.port(), targets[2], "must walk past both held ports"); + assert!(lease.is_held(), "the production probe must hold the socket"); + drop((first, second)); + } +} diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index c00872ed0..a5889f1ef 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -5,12 +5,13 @@ //! Spawn, attach to, and stream logs from a managed background service //! process, plus the endpoint-key/auth checks that gate it. //! -//! Mechanically relocated from `main.rs` with no behavior change. This module +//! Relocated from `main.rs`, since extended with the port-allocation and +//! launch-lock transaction. This module //! is the managed-service-spawning tail that was deliberately deferred out of //! the Phase 6a (ROCMAI-82) split: `start_managed_service`, //! `run_attached_service`, `spawn_managed_engine_child`, `stream_attached_logs`, //! and the rest of this file. It was entangled with crate-root launch helpers -//! — `record_cli_audit_event`, `existing_live_managed_service`, and the stop +//! — `record_cli_audit_event`, `existing_live_managed_service_in`, and the stop //! action of the service-record-management cluster (this module reaches only //! stop, via `AttachOutcome::Stop`; restart/remove/prune are not called here) //! — and this later pass (ROCMAI-91, Phase 6b) separates the spawning tail out @@ -21,7 +22,7 @@ //! `ServicesCommand`/`services()` stay in `main.rs` (a dedicated clap //! subcommand's enum and dispatch function usually do, per //! `docs/architecture.md`'s convention note). `record_cli_audit_event`, -//! `existing_live_managed_service`, and the broader "service record management" +//! `existing_live_managed_service_in`, and the broader "service record management" //! cluster (stop/restart/remove/prune, reached from `services()`; stop/restart //! are also reachable from the chat/LLM sandbox tool-calling path) stay at the //! crate root too — reached via `crate::`, same as other mechanically relocated @@ -34,8 +35,6 @@ use std::fs; use std::io::{self, BufRead, Write}; use std::path::{Path, PathBuf}; #[cfg(not(windows))] -use std::process::ExitStatus; -#[cfg(not(windows))] use std::process::{Command as ProcessCommand, Stdio}; use std::thread; use std::time::Duration; @@ -55,9 +54,11 @@ use crate::engine_runtime_select::{ managed_service_launcher_path, wait_for_service_http_ready_with_progress, }; use crate::engines_cmd::env_root_for_service; -use crate::managed_service_status::existing_live_managed_service; +use crate::managed_service_status::{ + existing_live_managed_service_in, load_managed_services, managed_service_is_live, +}; +use crate::serve_port; use crate::service_hints; -#[cfg(not(windows))] use crate::services_text::read_optional_tail_lines; use crate::services_text::{render_service_action_result, run_internal_sandbox_tool}; use crate::{SandboxToolArg, device_policy_name, record_cli_audit_event, status_for_readiness}; @@ -129,9 +130,9 @@ pub(crate) fn resolve_endpoint_auth( } } -/// When an equivalent managed service is already running, the endpoint key -/// serve() freshly stored for this attempt is unused — drop it so it is not -/// orphaned in storage. The already-running service keeps its own key. +/// When an equivalent managed service is already running, or the launch fails +/// before any record exists, the endpoint key serve() freshly stored for this +/// attempt is unused — drop it so it is not orphaned in storage. The already-running service keeps its own key. /// Best-effort and idempotent; a loopback attempt (`freshly_stored == None`) /// is a no-op. pub(crate) fn drop_orphaned_endpoint_key_on_already_running( @@ -212,7 +213,7 @@ pub(crate) fn ensure_public_service_has_endpoint_key( "managed service is bound to the public host `{host}` but has no endpoint API key, \ so restarting it would reopen it without authentication. The key is dropped when a \ service stops and cannot be recovered. Launch it again with \ - `rocm serve --host {host} --allow-public-bind` (add `--api-key `, or set \ + `rocm serve --host {host} --port --allow-public-bind` (add `--api-key `, or set \ ROCM_SERVE_API_KEY, to choose the key instead of generating one)." ); } @@ -244,8 +245,14 @@ pub(crate) fn attach_background_stdio( Ok(()) } -#[cfg(not(windows))] -pub(crate) fn managed_engine_startup_failure_detail(status: ExitStatus, log_path: &Path) -> String { +/// `status_clause` is a pre-formatted fragment inserted right after "exited +/// immediately" (e.g. `" with status exit status: 1"` on Unix, where a real +/// `ExitStatus` is available; empty on Windows, where the liveness check +/// below only has a yes/no answer, not an exit code). +pub(crate) fn managed_engine_startup_failure_detail( + status_clause: &str, + log_path: &Path, +) -> String { let mut recent_lines = read_optional_tail_lines(log_path, 80, "service log"); if recent_lines.is_empty() { for _ in 0..5 { @@ -258,12 +265,12 @@ pub(crate) fn managed_engine_startup_failure_detail(status: ExitStatus, log_path } if recent_lines.is_empty() { return format!( - "managed engine exited immediately with status {status}; inspect {}", + "managed engine exited immediately{status_clause}; inspect {}", log_path.display() ); } format!( - "managed engine exited immediately with status {status}; inspect {}\n\nrecent startup log output:\n{}", + "managed engine exited immediately{status_clause}; inspect {}\n\nrecent startup log output:\n{}", log_path.display(), recent_lines.join("\n") ) @@ -310,11 +317,148 @@ enum ManagedSpawn { }, } +/// Outcome of the locked allocation step. +/// +/// `Debug` so tests can assert on `Result::expect_err`. +#[derive(Debug)] +enum ManagedPortDecision { + /// An equivalent managed service is already live; nothing may be spawned. + /// Boxed to keep the variants a similar size (clippy::large_enum_variant). + AlreadyLive(Box), + /// The concrete port this launch owns until it hands it to the child. + Lease(rocm_core::LoopbackPortLease), +} + +/// Live managed services, with liveness refreshed by [`load_managed_services`] +/// (dead PIDs demote to "stopped" and therefore neither satisfy the duplicate +/// check nor reserve a port). +/// +/// A read failure degrades to "nothing is live": for an already-`ready` +/// service the OS preflight still guards its port (the engine holds it), and +/// refusing to serve because the services directory could not be listed would +/// be a worse failure than launching. That guard does not cover a `starting` +/// record: its port is released back to the OS in +/// `launch_managed_engine_child_locked` (see `lease.release()`), right before the +/// spawn, and stays unbound until the new engine child has actually bound it, so a concurrent launch that hits this +/// read failure in that narrow window can probe the same port successfully. +/// Accepted as a known gap — narrower than the general services-dir read +/// failure case, and only reachable by a second, near-simultaneous I/O +/// failure on top of one already in flight. +fn live_managed_services(paths: &AppPaths) -> Vec { + load_managed_services(paths) + .unwrap_or_default() + .into_iter() + .filter(managed_service_is_live) + .collect() +} + +/// Ports that live managed services still own. +/// +/// Conservative by design: a record reserves its port regardless of the host it +/// recorded, because a `0.0.0.0` (or `localhost`) listener also blocks the +/// loopback address a new local server would bind. Over-reserving costs one +/// candidate in a 101-port scan; under-reserving costs a collision. +fn reserved_service_ports(live: &[ManagedServiceRecord]) -> Vec { + live.iter() + .map(|record| serve_port::ReservedPort { + port: record.port, + service_id: record.service_id.clone(), + status: record.status.clone(), + }) + .collect() +} + +/// The decision half of the launch transaction: refresh liveness, apply the +/// `(engine, canonical_model_id)` idempotency guard, then lease a concrete port +/// against both live records and the real OS. +/// +/// Callers MUST hold [`rocm_core::AppPaths::managed_launch_lock_path`] across +/// this call *and* the record publication that follows, or two launches can +/// pick the same candidate. +/// `probe` is injected so tests can drive deterministic bind outcomes without a +/// GPU or a spawn. +fn resolve_managed_port_in_transaction( + paths: &AppPaths, + engine: &str, + canonical_model_id: &str, + host: &str, + request: serve_port::PortRequest, + probe: &mut dyn FnMut(u16) -> std::io::Result, +) -> Result { + let live = live_managed_services(paths); + // Idempotency first: an already-live equivalent service means no port is + // needed at all, so it must be answered before any candidate is leased. + if let Some(existing) = existing_live_managed_service_in(&live, engine, canonical_model_id) { + return Ok(ManagedPortDecision::AlreadyLive(Box::new(existing.clone()))); + } + let reserved = reserved_service_ports(&live); + serve_port::resolve_serve_port(host, request, &reserved, probe).map(ManagedPortDecision::Lease) +} + +/// Grace given to a child that must be terminated because its record could not +/// be published. Short: the child is milliseconds old and has nothing to flush. +const MANAGED_LAUNCH_CLEANUP_GRACE: Duration = Duration::from_secs(2); + +/// Undo a failed managed launch attempt before the launch lock is released, +/// so no failed transaction leaves a live reservation or a stray endpoint key. +/// +/// Order matters: a spawned child is terminated *first*, so it is not still +/// holding the port or writing to the log by the time cleanup runs. The +/// termination outcome is audited but not acted on: the record is marked +/// `failed` (releasing the port) even when death was not confirmed. If no child was ever spawned (`record.engine_pid` is still `None`), +/// every artifact this attempt created is removed: there is nothing for +/// `rocm services`/`rocm logs` to show. If a child did run, its record and log +/// are kept instead (marked `status = "failed"`), since the error that led here +/// already told the user to go inspect that log; only the endpoint key, which a +/// failed attempt has no use for, is cleared. +/// Every step is best-effort and idempotent; cleanup must never mask the +/// original failure with a second one. +fn abandon_failed_managed_launch(paths: &AppPaths, record: &ManagedServiceRecord) { + if let Some(pid) = record.engine_pid.filter(|pid| *pid != 0) { + let outcome = rocm_core::terminate_verified( + &rocm_core::ProcessIdentity::capture(pid), + rocm_core::KillScope::Tree, + MANAGED_LAUNCH_CLEANUP_GRACE, + true, + ); + record_cli_audit_event( + paths, + "service", + "managed_service_launch_cleanup", + "warn", + format!( + "terminated engine child pid={pid} after a failed launch publication outcome={}", + outcome.as_str() + ), + Some(&record.service_id), + ); + let mut failed = record.clone(); + failed.status = "failed".to_owned(); + let _ = failed.write(); + endpoint_keys::clear_endpoint_api_key(paths, &record.service_id); + return; + } + for path in [ + record.manifest_path.as_path(), + record.log_path.as_path(), + record.engine_state_path.as_path(), + ] { + let _ = fs::remove_file(path); + } + endpoint_keys::clear_endpoint_api_key(paths, &record.service_id); +} + /// Spawn the detached engine child shared by the managed (background) and /// attached (`--verbose`/`--foreground`) serve paths. Returns before the HTTP /// readiness wait; callers decide whether to block on readiness /// ([`start_managed_service`]) or start tailing the log immediately /// ([`run_attached_service`]). +/// +/// The whole body is one serialized transaction under the shared allocation +/// lock: duplicate detection, live-reservation refresh, candidate leasing, +/// manifest/log creation, argv preparation, the bounded immediate-exit check, +/// and record publication. Readiness deliberately runs after the lock is +/// released. #[allow(clippy::too_many_arguments)] fn spawn_managed_engine_child( paths: &AppPaths, @@ -323,7 +467,7 @@ fn spawn_managed_engine_child( requested_model: &str, resolve: &ResolveModelResponse, host: &str, - port: u16, + port_request: serve_port::PortRequest, device_policy: &DevicePolicy, gpu_indices: &[u32], runtime_id: Option<&str>, @@ -334,6 +478,14 @@ fn spawn_managed_engine_child( paths.ensure()?; fs::create_dir_all(paths.services_dir())?; + // No lock is acquired here: every production caller + // ([`start_managed_service`], [`run_attached_service`]) already holds + // [`rocm_core::AppPaths::managed_launch_lock_path`] for the whole of this + // call, so no two transactions can choose the same port or publish + // competing records for one engine+model. A second acquisition in this + // frame would self-deadlock, since the lock is exclusive per open file + // description, not per process. + // // Idempotency guard: if a managed service for this engine+model is already // alive, surface it and spawn nothing. A second `serve --managed` (e.g. the // chat assistant re-issuing the same request) is treated as satisfied, not @@ -344,63 +496,85 @@ fn spawn_managed_engine_child( .map(serde_json::to_string) .transpose() .context("failed to encode engine recipe hint")?; - if let Some(existing) = - existing_live_managed_service(paths, engine, &resolve.canonical_model_id) - { - if existing.engine_recipe_json != requested_recipe_json { - bail!( - "managed service `{}` is already running for engine `{engine}` and model `{}` with different serve options (recipe hint, tool-call parser, or generation defaults); stop it and run `rocm serve` again to apply the requested options", - existing.service_id, - resolve.canonical_model_id - ); - } - // Reuse cannot satisfy a demand for auth the running server never got. - // The engine reads its key once, at launch, from the environment this - // function builds below — so a server started without one keeps serving - // anonymously no matter what is written afterwards. Upgrading the record - // here would be worse than doing nothing: the record would claim auth - // that the live process does not enforce, and - // `ensure_public_service_has_endpoint_key` would pass on the strength of - // a key file nothing reads. - // - // This is what `rocm remote serve` relies on. It publishes a loopback - // port onto the tailnet and prints "the API key above is what stops - // anyone else calling it". Reusing an unauthenticated service silently - // would make that sentence false about an endpoint the whole tailnet can - // reach. Refusing is the only answer that fails closed, and it is the - // same shape as the recipe mismatch above. - if require_api_key && !existing.requires_api_key { - bail!( - "managed service `{}` is already running for engine `{engine}` and model `{}` \ - without authentication, and a running server cannot be given a key it did not \ - start with; stop it with `{}` and run the command again to \ - serve it with `--require-api-key`", - existing.service_id, - resolve.canonical_model_id, - service_hints::service_stop_hint(&existing.service_id) + let lease = match resolve_managed_port_in_transaction( + paths, + engine, + &resolve.canonical_model_id, + host, + port_request, + &mut serve_port::loopback_bind_probe, + )? { + ManagedPortDecision::AlreadyLive(existing) => { + let existing = *existing; + if existing.engine_recipe_json != requested_recipe_json { + bail!( + "managed service `{}` is already running for engine `{engine}` and model `{}` with different serve options (recipe hint, tool-call parser, or generation defaults); stop it and run `rocm serve` again to apply the requested options", + existing.service_id, + resolve.canonical_model_id + ); + } + // An explicit port is a demand, not a hint: reporting success on a + // different port than the one asked for would silently drop it. + if let Some(port) = port_request.explicit() + && port != existing.port + { + bail!( + "managed service `{}` is already serving engine `{engine}` and model `{}` on port {}, so port {port} would not be used; stop it with `{}` and run the command again to serve on that port", + existing.service_id, + resolve.canonical_model_id, + existing.port, + service_hints::service_stop_hint(&existing.service_id) + ); + } + // Reuse cannot satisfy a demand for auth the running server never got. + // The engine reads its key once, at launch, from the environment this + // function builds below — so a server started without one keeps serving + // anonymously no matter what is written afterwards. Upgrading the record + // here would be worse than doing nothing: the record would claim auth + // that the live process does not enforce, and + // `ensure_public_service_has_endpoint_key` would pass on the strength of + // a key file nothing reads. + // + // This is what `rocm remote serve` relies on. It publishes a loopback + // port onto the tailnet and prints "the API key above is what stops + // anyone else calling it". Reusing an unauthenticated service silently + // would make that sentence false about an endpoint the whole tailnet can + // reach. Refusing is the only answer that fails closed, and it is the + // same shape as the recipe mismatch above. + if require_api_key && !existing.requires_api_key { + bail!( + "managed service `{}` is already running for engine `{engine}` and model `{}` \ + without authentication, and a running server cannot be given a key it did not \ + start with; stop it with `{}` and run the command again to \ + serve it with `--require-api-key`", + existing.service_id, + resolve.canonical_model_id, + service_hints::service_stop_hint(&existing.service_id) + ); + } + record_cli_audit_event( + paths, + "service", + "managed_service_launch_skipped", + "info", + format!( + "skipped duplicate managed launch engine={engine} model={} existing_service_id={} status={}", + resolve.canonical_model_id, existing.service_id, existing.status + ), + Some(&existing.service_id), ); + return Ok(ManagedSpawn::AlreadyRunning(ManagedLaunchReport { + service_id: existing.service_id, + endpoint_url: existing.endpoint_url, + status: existing.status, + already_running: true, + child_pid: None, + log_path: None, + manifest_path: None, + })); } - record_cli_audit_event( - paths, - "service", - "managed_service_launch_skipped", - "info", - format!( - "skipped duplicate managed launch engine={engine} model={} existing_service_id={} status={}", - resolve.canonical_model_id, existing.service_id, existing.status - ), - Some(&existing.service_id), - ); - return Ok(ManagedSpawn::AlreadyRunning(ManagedLaunchReport { - service_id: existing.service_id, - endpoint_url: existing.endpoint_url, - status: existing.status, - already_running: true, - child_pid: None, - log_path: None, - manifest_path: None, - })); - } + ManagedPortDecision::Lease(lease) => lease, + }; let mut record = ManagedServiceRecord::new( paths, @@ -409,7 +583,7 @@ fn spawn_managed_engine_child( requested_model, resolve.canonical_model_id.clone(), host, - port, + lease.port(), "managed", 0, runtime_id.map(str::to_owned), @@ -432,8 +606,85 @@ fn spawn_managed_engine_child( // This field means "the user demanded auth on a bind that would not otherwise // require it". A public bind needs no such record; its address still says so. record.requires_api_key = require_api_key; + // Publishing the record inside the lock is what reserves the leased port for + // every other transaction. From here on, any failure must remove it again. record.write()?; + match launch_managed_engine_child_locked( + paths, + &mut record, + lease, + engine, + service_id, + resolve, + host, + device_policy, + gpu_indices, + runtime_id, + env_id, + engine_recipe, + #[cfg(not(windows))] + None, + ) { + Ok(child_pid) => Ok(ManagedSpawn::Spawned { + record: Box::new(record), + child_pid, + }), + Err(error) => { + // The single cleanup choke point for every failure path below the + // record's first write: `record.engine_pid` tells it whether a child + // ever ran, so a pre-spawn failure deletes everything and a post-spawn + // one keeps the record and log (marked `status = "failed"`) instead. + abandon_failed_managed_launch(paths, &record); + Err(error) + } + } +} + +// Test-only, thread-scoped spawn interceptor for callers that cannot take a +// `spawn_override` argument (`start_managed_service`, `run_attached_service`): +// lets a test observe state at the instant production would spawn the child. +#[cfg(all(test, not(windows)))] +thread_local! { + static TEST_SPAWN_HOOK: std::cell::RefCell>>> = + const { std::cell::RefCell::new(None) }; +} + +#[cfg(all(test, not(windows)))] +fn test_spawn_hook() -> Option>> { + TEST_SPAWN_HOOK.with(|hook| hook.borrow().clone()) +} + +#[cfg(not(windows))] +type SpawnOverride<'a> = dyn Fn(&mut ProcessCommand) -> io::Result + 'a; + +/// The side-effecting half of the launch transaction: create the log, prepare +/// concrete argv, hand the leased port to the child, spawn it, reject a child +/// that has already exited, and publish the running record. +/// +/// Split out of [`spawn_managed_engine_child`] so that *every* failure below the +/// record's first write is funnelled through one cleanup path before the +/// launch lock is released. Returns the child pid. +#[allow(clippy::too_many_arguments)] +fn launch_managed_engine_child_locked( + paths: &AppPaths, + record: &mut ManagedServiceRecord, + lease: rocm_core::LoopbackPortLease, + engine: &str, + service_id: &str, + resolve: &ResolveModelResponse, + host: &str, + device_policy: &DevicePolicy, + gpu_indices: &[u32], + runtime_id: Option<&str>, + env_id: Option<&str>, + engine_recipe: Option<&EngineRecipeHint>, + // Test-only seam: lets a test assert the leased port is actually bindable + // (i.e. released) at the exact point production would spawn the child, + // without needing to fake or intercept the real spawn mechanics. + #[cfg(not(windows))] spawn_override: Option<&SpawnOverride<'_>>, +) -> Result { + let port = record.port; if let Some(parent) = record.engine_state_path.parent() { fs::create_dir_all(parent) .with_context(|| format!("failed to create {}", parent.display()))?; @@ -485,6 +736,12 @@ fn spawn_managed_engine_child( endpoint_key_file.is_some(), record.requires_api_key, )?; + // Hand the port over: close the probe socket immediately before the spawn, + // still under the caller's launch lock, so no other ROCm transaction can + // take it in the meantime. On a custom host the lease never held a socket + // and this is a no-op — the engine owns that bind. + debug_assert_eq!(lease.port(), port); + lease.release(); #[cfg(windows)] let child_pid = { let env_values = app_path_env_var_values(paths, engine_envs_root.as_deref()); @@ -492,8 +749,19 @@ fn spawn_managed_engine_child( if let Some(key_file) = endpoint_key_file.as_deref() { env_refs.push((rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV, key_file)); } - rocm_core::spawn_detached_no_inherit(¤t_exe, &serve_args, &env_refs) - .context("failed to launch managed engine process")? + let child_pid = rocm_core::spawn_detached_no_inherit(¤t_exe, &serve_args, &env_refs) + .context("failed to launch managed engine process")?; + // Set before the immediate-exit check below, so a bail! here still leaves + // `record.engine_pid` telling cleanup that a child was spawned. + record.engine_pid = Some(child_pid); + thread::sleep(Duration::from_millis(200)); + if !process_is_running(child_pid) { + bail!( + "{}", + managed_engine_startup_failure_detail("", &record.log_path) + ); + } + child_pid }; #[cfg(not(windows))] let child_pid = { @@ -508,10 +776,19 @@ fn spawn_managed_engine_child( if let Some(key_file) = endpoint_key_file.as_deref() { command.env(rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV, key_file); } - let mut child = command - .spawn() - .context("failed to launch managed engine process")?; + #[cfg(all(test, not(windows)))] + let hook = test_spawn_hook(); + #[cfg(all(test, not(windows)))] + let spawn_override = spawn_override.or(hook.as_deref()); + let mut child = match spawn_override { + Some(spawn) => spawn(&mut command), + None => command.spawn(), + } + .context("failed to launch managed engine process")?; let child_pid = child.id(); + // Set before the immediate-exit check below, so a bail! here still leaves + // `record.engine_pid` telling cleanup that a child was spawned. + record.engine_pid = Some(child_pid); thread::sleep(Duration::from_millis(200)); if let Some(status) = child .try_wait() @@ -519,7 +796,10 @@ fn spawn_managed_engine_child( { bail!( "{}", - managed_engine_startup_failure_detail(status, &record.log_path) + managed_engine_startup_failure_detail( + &format!(" with status {status}"), + &record.log_path + ) ); } child_pid @@ -532,22 +812,38 @@ fn spawn_managed_engine_child( record.record_supervisor_identity(child_pid); record.record_engine_identity(child_pid); record.status = "running".to_owned(); - record.write()?; + if let Err(error) = record.write() { + // Cleanup is the caller's job now: `record.engine_pid` (set above) is + // already enough for `abandon_failed_managed_launch` to terminate this + // child, so there is nothing left for this inner path to do beyond + // surfacing the error. + return Err(error.context("failed to publish the managed service record")); + } - Ok(ManagedSpawn::Spawned { - record: Box::new(record), - child_pid, - }) + Ok(child_pid) } +/// Launch a managed service in the background: run the locked allocation and +/// spawn transaction, release the launch lock once the record is published, +/// then wait for HTTP readiness. +/// +/// Drops the endpoint key `serve()` freshly staged for this attempt if the +/// launch fails before any `ManagedServiceRecord` exists to own it (port +/// reserved/in use, range exhausted, other bind error) — such a failure +/// returns before `abandon_failed_managed_launch` ever runs, so nothing else +/// would clear it. Scoped to the spawn call only, mirroring +/// [`run_attached_service`]'s identical guard: a later failure (the readiness +/// wait, `record.write()`) leaves a record a live child actually owns, so it +/// must not also wipe that key. #[allow(clippy::too_many_arguments)] pub(crate) fn start_managed_service( + paths: &AppPaths, engine: &str, service_id: &str, requested_model: &str, resolve: &ResolveModelResponse, host: &str, - port: u16, + port_request: serve_port::PortRequest, device_policy: &DevicePolicy, gpu_indices: &[u32], runtime_id: Option<&str>, @@ -558,25 +854,30 @@ pub(crate) fn start_managed_service( require_api_key: bool, on_wait_tick: &mut dyn FnMut(Duration), ) -> Result { - let paths = AppPaths::discover()?; let (mut record, child_pid) = match spawn_managed_engine_child( - &paths, + paths, engine, service_id, requested_model, resolve, host, - port, + port_request, device_policy, gpu_indices, runtime_id, env_id, engine_recipe, require_api_key, - )? { + ) + .inspect_err(|_| { + drop_orphaned_endpoint_key_on_already_running(paths, service_id, endpoint_api_key); + })? { ManagedSpawn::AlreadyRunning(report) => return Ok(report), ManagedSpawn::Spawned { record, child_pid } => (*record, child_pid), }; + // The transaction resolved the request to one concrete port; everything + // below — readiness, the endpoint URL, the audit line — uses that. + let port = record.port; // The claiming service record is now persisted, so the selected GPU is // visible to any concurrent auto-selection. Release the launch lock before // the readiness wait below, which can block for many seconds — holding it @@ -609,7 +910,7 @@ pub(crate) fn start_managed_service( record.write()?; let endpoint_url = format!("{}/v1", format_http_base_url(host, port)); record_cli_audit_event( - &paths, + paths, "service", "managed_service_launch", "info", @@ -873,12 +1174,13 @@ enum AttachOutcome { /// survives the session — Ctrl-D detaches and leaves it running, Ctrl-C stops it. #[allow(clippy::too_many_arguments)] pub(crate) fn run_attached_service( + paths: &AppPaths, engine: &str, service_id: &str, requested_model: &str, resolve: &ResolveModelResponse, host: &str, - port: u16, + port_request: serve_port::PortRequest, gpu_indices: &[u32], runtime_id: Option<&str>, env_id: Option<&str>, @@ -886,29 +1188,36 @@ pub(crate) fn run_attached_service( launch_lock: rocm_core::FileLock, require_api_key: bool, ) -> Result<()> { - let paths = AppPaths::discover()?; - let spawn = spawn_managed_engine_child( - &paths, + paths, engine, service_id, requested_model, resolve, host, - port, + port_request, &resolve.device_policy, gpu_indices, runtime_id, env_id, resolve.engine_recipe.as_ref(), require_api_key, - )?; + ) + // Mirrors `start_managed_service`'s cleanup: a pre-spawn + // failure (port reserved/in use, range exhausted, other bind error) returns + // before any service record exists, so nothing else clears the key just + // stored for this attempt. + .inspect_err(|_| { + drop_orphaned_endpoint_key_on_already_running(paths, service_id, endpoint_api_key); + })?; // The claiming record is persisted (or an existing service was found), so the // selected GPU is now visible to concurrent auto-selection. Release the launch // lock before streaming logs, which blocks for the whole attached session. drop(launch_lock); - let (service_id, log_path, child_pid) = match spawn { + // Attached mode is a managed record too, so it shares the transaction's + // resolved concrete port. + let (service_id, log_path, port, child_pid) = match spawn { // A server for this engine+model is already live. Don't fight it for the // port — point the user at the existing one instead of tailing a log we // did not start. @@ -922,12 +1231,15 @@ pub(crate) fn run_attached_service( " stop: {}", service_hints::service_stop_hint(&report.service_id) ); - drop_orphaned_endpoint_key_on_already_running(&paths, service_id, endpoint_api_key); + drop_orphaned_endpoint_key_on_already_running(paths, service_id, endpoint_api_key); return Ok(()); } - ManagedSpawn::Spawned { record, child_pid } => { - (service_id.to_owned(), record.log_path.clone(), child_pid) - } + ManagedSpawn::Spawned { record, child_pid } => ( + service_id.to_owned(), + record.log_path.clone(), + record.port, + child_pid, + ), }; // The child is a managed service that outlives this session once detached, so @@ -965,7 +1277,7 @@ pub(crate) fn run_attached_service( println!(" logs: rocm logs {service_id}"); println!(" stop: {}", service_hints::service_stop_hint(&service_id)); record_cli_audit_event( - &paths, + paths, "service", "serve_detach", "info", @@ -977,7 +1289,7 @@ pub(crate) fn run_attached_service( AttachOutcome::Stop => { println!("stopping server..."); match run_internal_sandbox_tool( - &paths, + paths, SandboxToolArg::StopServer, Some(service_id.clone()), true, @@ -991,7 +1303,7 @@ pub(crate) fn run_attached_service( } } record_cli_audit_event( - &paths, + paths, "service", "serve_stop", "info", @@ -1160,6 +1472,7 @@ fn stream_attached_logs_no_tty(log_path: &Path, child_pid: u32) -> Result Result<()> { + let (root, paths) = test_paths("dup-managed-explicit-port-differs"); + paths.ensure()?; + let mut existing = ManagedServiceRecord::new( + &paths, + "lemonade-qwen-1000", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + 11510, + "managed", + std::process::id(), + None, + None, + None, + ); + existing.status = "ready".to_owned(); + existing.engine_pid = Some(std::process::id()); + existing.write()?; + + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + served_model_id: None, + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + let launch = |port_request| { + spawn_managed_engine_child( + &paths, + "lemonade", + "lemonade-qwen-2000", + "qwen", + &resolve, + "127.0.0.1", + port_request, + &resolve.device_policy, + &[], + None, + None, + None, + false, + ) + }; + + let different = launch(serve_port::PortRequest::Explicit(11511)); + // The same port, and no port at all, still reuse the live service. + let same = launch(serve_port::PortRequest::Explicit(11510)); + let auto = launch(serve_port::PortRequest::Auto); + let _ = fs::remove_dir_all(root); + + let Err(error) = different else { + panic!("a port the live service does not serve must be refused") + }; + let message = error.to_string(); + assert!( + message.contains("on port 11510") && message.contains("port 11511 would not be used"), + "{message}" + ); + assert!(matches!(same?, ManagedSpawn::AlreadyRunning(_))); + assert!(matches!(auto?, ManagedSpawn::AlreadyRunning(_))); + Ok(()) + } + #[test] fn spawn_managed_engine_child_blocks_reuse_with_mismatched_recipe() -> Result<()> { // A live service recorded with one recipe (e.g. a tool-call parser flag) @@ -1253,7 +1642,7 @@ mod tests { "qwen", &resolve, "127.0.0.1", - 11511, + serve_port::PortRequest::Explicit(11511), &resolve.device_policy, &[], None, @@ -1337,7 +1726,7 @@ mod tests { "qwen", &resolve, "127.0.0.1", - 11520, + serve_port::PortRequest::Explicit(11520), &resolve.device_policy, &[], None, @@ -1382,7 +1771,7 @@ mod tests { fn a_managed_spawn_refuses_an_invalid_key_file_on_a_service_that_requires_one() -> Result<()> { // Drives the real call site, not the guard's own arguments. The service // was launched with `--require-api-key`, and its key file is present but - // empty — so `requires_api_key` is true while `key_present` is false, + // invalid — so `requires_api_key` is true while `key_present` is false, // which is the only way to reach the first branch. A test calling // `ensure_public_service_has_endpoint_key` directly cannot catch a // mis-wired call site, which is the defect that has occurred here twice. @@ -1394,9 +1783,18 @@ mod tests { paths.ensure()?; fs::create_dir_all(paths.services_dir())?; - // Empty, so the file exists (requires_api_key = true) but yields no - // usable key (key_present = false). - endpoint_keys::store_endpoint_api_key(&paths, "lemonade-qwen-3000", "")?; + // Non-empty but invalid (embedded CR/LF), so the file exists and is not + // merely empty (requires_api_key = true) but yields no usable key + // (key_present = false) — `endpoint_api_key_from_file` rejects embedded + // control characters. An empty seed here would let the post-condition + // below pass even if the real cleanup call never ran, since + // `endpoint_api_key` already reads back `None` for an empty file whether + // or not it was ever deleted. + endpoint_keys::store_endpoint_api_key( + &paths, + "lemonade-qwen-3000", + "bad-key\r\nX-Injected: value", + )?; let resolve = ResolveModelResponse { canonical_model_id: "qwen-canonical".to_owned(), @@ -1424,7 +1822,7 @@ mod tests { // Loopback on purpose: the public-bind branch must not be what // refuses this, or the test would pass with the guard disabled. "127.0.0.1", - 11512, + serve_port::PortRequest::Explicit(11512), &resolve.device_policy, &[], None, @@ -1432,6 +1830,18 @@ mod tests { None, true, ); + + // This guard fires before any child is spawned, so `record.engine_pid` + // is still `None` when `spawn_managed_engine_child`'s cleanup arm calls + // `abandon_failed_managed_launch` — proving its pre-spawn branch + // (delete everything) runs from the real call site, not just from the + // `abandoned_launch_leaves_no_record_log_or_endpoint_key` test that + // calls the helper directly. + let manifest_path = paths.service_manifest_path("lemonade-qwen-3000"); + let log_path = paths.service_log_path("lemonade-qwen-3000"); + let engine_state_path = paths.service_engine_state_path("lemonade", "lemonade-qwen-3000"); + let key_file_path = endpoint_keys::endpoint_key_file_path(&paths, "lemonade-qwen-3000"); + let live = live_managed_services(&paths); let _ = fs::remove_dir_all(root); let Err(error) = result else { @@ -1446,56 +1856,453 @@ mod tests { message.contains("without authentication"), "the refusal must say what the risk is: {message}" ); + assert!( + !manifest_path.exists(), + "a pre-spawn failure must not leave a record" + ); + assert!( + !log_path.exists(), + "a pre-spawn failure must not leave a log" + ); + assert!( + !engine_state_path.exists(), + "a pre-spawn failure must not leave engine state" + ); + assert!( + !key_file_path.exists(), + "a pre-spawn failure must delete the unusable key file from disk, not merely \ + leave it unreadable as a key" + ); + assert!( + live.is_empty(), + "a pre-spawn failure must leave no reservation" + ); Ok(()) } #[test] - fn resolve_endpoint_auth_loopback_stays_credential_free() { - // Loopback binds never require auth, even if a key is supplied. - for host in ["127.0.0.1", "localhost", "::1"] { - assert_eq!(resolve_endpoint_auth(host, None, false).unwrap(), None); - assert_eq!( - resolve_endpoint_auth(host, Some("ignored"), false).unwrap(), - None + fn a_failed_spawn_past_the_first_write_terminates_the_child_and_keeps_a_failed_record() + -> Result<()> { + // Drives the real call site's *post*-spawn cleanup branch, which + // neither `abandoned_launch_*` test reaches (they call + // `abandon_failed_managed_launch` directly). An engine name this + // binary does not build in makes the real spawned child (a second + // `rocm` process) exit almost immediately via + // `run_builtin_engine_serve_http`'s `other => bail!(...)` arm, after + // `launch_managed_engine_child_locked` has already set + // `record.engine_pid` — so cleanup must terminate (already-dead) the + // child, keep the record and log for inspection, mark the record + // `failed`, and clear only the endpoint key. + // + // The child always dies near-instantly, but `spawn_managed_engine_child` + // only reports that within a fixed startup-check window. On a loaded or + // antivirus-scanning Windows CI box, a fresh debug `rocm.exe` can + // occasionally be slow enough to start that the window is missed even + // though the child is already doomed — so poll by retrying a fresh + // attempt (its own service id/port, so no stale manifest collides) up + // to a deadline instead of asserting on one race-prone attempt. + let deadline = std::time::Instant::now() + Duration::from_secs(10); + let mut attempt: u16 = 0; + let (root, paths, service_id, result) = loop { + attempt += 1; + let (root, paths) = + test_paths(&format!("managed-spawn-unknown-engine-cleanup-{attempt}")); + paths.ensure()?; + let service_id = format!("no-such-engine-qwen-{}", 4000 + attempt); + endpoint_keys::store_endpoint_api_key(&paths, &service_id, "test-key")?; + + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + served_model_id: None, + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + + let result = spawn_managed_engine_child( + &paths, + "no-such-engine", + &service_id, + "qwen", + &resolve, + "127.0.0.1", + serve_port::PortRequest::Explicit(11_514 + attempt), + &resolve.device_policy, + &[], + None, + None, + None, + false, ); - } - } - - #[test] - fn resolve_endpoint_auth_loopback_can_be_required_when_something_republishes_it() { - // "Loopback" describes the bind address, not who can reach the port. A - // tailnet publish, a proxy, or a container port map all leave the bind - // loopback while widening the audience, and the default policy would - // hand out an unauthenticated endpoint. Whoever widens the reach asks - // for the credential explicitly. - for host in ["127.0.0.1", "localhost", "::1"] { - let generated = resolve_endpoint_auth(host, None, true) - .unwrap() - .expect("a required key must be generated, not skipped"); - assert!(!generated.trim().is_empty()); - assert_eq!( - resolve_endpoint_auth(host, Some("supplied-key"), true).unwrap(), - Some("supplied-key".to_owned()), - "a supplied key must be honoured rather than ignored as it is by default" - ); - } + if result.is_ok() && std::time::Instant::now() < deadline { + let _ = fs::remove_dir_all(&root); + continue; + } + break (root, paths, service_id, result); + }; - // The same validation a public bind gets: an empty key is a refusal, not - // a silent downgrade to no auth. - assert!(resolve_endpoint_auth("127.0.0.1", Some(" "), true).is_err()); - } + let manifest_path = paths.service_manifest_path(&service_id); + let log_path = paths.service_log_path(&service_id); + let persisted = load_managed_services(&paths) + .expect("re-read manifests") + .into_iter() + .find(|candidate| candidate.service_id == service_id); + let key = endpoint_keys::endpoint_api_key(&paths, &service_id); + let live = live_managed_services(&paths); - #[test] - fn resolve_endpoint_auth_public_uses_supplied_key_trimmed() { - let key = resolve_endpoint_auth("0.0.0.0", Some(" my-key "), false) - .unwrap() - .expect("public bind must have a key"); - assert_eq!(key, "my-key"); + let Err(error) = result else { + panic!("an engine this binary does not build in must not report success") + }; + let message = error.to_string(); + assert!( + message.contains("managed engine exited immediately"), + "the failure must come from the spawned child dying, not an earlier guard: {message}" + ); + assert!( + manifest_path.exists(), + "a post-spawn failure must keep the record for inspection" + ); + assert!( + log_path.exists(), + "a post-spawn failure must keep the log for inspection" + ); + let persisted = + persisted.expect("a post-spawn failure must leave the record on disk, not delete it"); + assert_eq!( + persisted.status, "failed", + "a post-spawn failure must mark the record failed, not leave it stuck" + ); + assert_eq!( + key, None, + "a post-spawn failure must still clear the endpoint key" + ); + assert!( + live.is_empty(), + "a failed record must not still reserve its port" + ); + let _ = fs::remove_dir_all(root); + Ok(()) } #[test] - fn resolve_endpoint_auth_public_generates_key_when_absent() { + fn failed_port_reservation_leaves_no_orphaned_endpoint_key() -> Result<()> { + // serve() stores the new attempt's endpoint key before calling + // start_managed_service. If spawn_managed_engine_child fails inside + // resolve_managed_port_in_transaction (explicit port already reserved + // by another live service), it returns before any + // ManagedServiceRecord exists for this attempt, so + // abandon_failed_managed_launch never runs. Driving the real function + // end to end (not a simulated call to + // drop_orphaned_endpoint_key_on_already_running) proves + // `start_managed_service`'s own `.inspect_err` actually wires the cleanup, mirroring how + // attached_launch_failing_before_spawn_leaves_no_orphaned_endpoint_key + // drives run_attached_service for the foreground path. + let (root, paths) = test_paths("orphaned-key-on-reserved-port"); + paths.ensure()?; + let mut other = ManagedServiceRecord::new( + &paths, + "lemonade-llama-1000", + "lemonade", + "llama", + "llama-canonical", + "127.0.0.1", + 11513, + "managed", + std::process::id(), + None, + None, + None, + ); + other.status = "ready".to_owned(); + other.engine_pid = Some(std::process::id()); + other.write()?; + + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + served_model_id: None, + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + + let service_id = "lemonade-qwen-2000"; + endpoint_keys::store_endpoint_api_key(&paths, service_id, "test-key")?; + let launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; + + let result = start_managed_service( + &paths, + "lemonade", + service_id, + "qwen", + &resolve, + "127.0.0.1", + serve_port::PortRequest::Explicit(11513), + &resolve.device_policy, + &[], + None, + None, + None, + Some("test-key"), + launch_lock, + false, + &mut |_elapsed| {}, + ); + let Err(error) = result else { + let _ = fs::remove_dir_all(root); + panic!("explicit port already reserved by a live service must be rejected") + }; + assert!( + error + .to_string() + .contains("already reserved by managed service"), + "unexpected error: {error}" + ); + + assert!( + endpoint_keys::endpoint_api_key(&paths, service_id).is_none(), + "cleanup must drop the key this failed attempt staged before launching" + ); + + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn attached_launch_failing_before_spawn_leaves_no_orphaned_endpoint_key() -> Result<()> { + // The background path's cleanup is the `.inspect_err` in + // start_managed_service. run_attached_service is the other caller of + // spawn_managed_engine_child and needs the identical .inspect_err() + // cleanup, not a simulated one: drive the real function end to end. + let (root, paths) = test_paths("orphaned-key-attached-reserved-port"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + let mut other = ManagedServiceRecord::new( + &paths, + "lemonade-llama-1000", + "lemonade", + "llama", + "llama-canonical", + "127.0.0.1", + 11514, + "managed", + std::process::id(), + None, + None, + None, + ); + other.status = "ready".to_owned(); + other.engine_pid = Some(std::process::id()); + other.write()?; + + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + served_model_id: None, + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + + let service_id = "lemonade-qwen-attached-2000"; + endpoint_keys::store_endpoint_api_key(&paths, service_id, "test-key")?; + let launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; + + let result = run_attached_service( + &paths, + "lemonade", + service_id, + "qwen", + &resolve, + "127.0.0.1", + serve_port::PortRequest::Explicit(11514), + &[], + None, + None, + Some("test-key"), + launch_lock, + false, + ); + let Err(error) = result else { + let _ = fs::remove_dir_all(root); + panic!("explicit port already reserved by a live service must be rejected") + }; + assert!( + error + .to_string() + .contains("already reserved by managed service"), + "unexpected error: {error}" + ); + assert!( + endpoint_keys::endpoint_api_key(&paths, service_id).is_none(), + "a pre-spawn failure must drop the key this attempt staged before launching" + ); + + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// The launch lock must still be held when the child is spawned: it is what + /// serializes allocation against publication. Observed through the spawn + /// hook, since releasing it early changes nothing else a test can see. + #[cfg(not(windows))] + #[test] + fn start_and_attached_launches_hold_the_launch_lock_until_the_spawn() -> Result<()> { + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + served_model_id: None, + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + for attached in [false, true] { + let (root, paths) = test_paths("launch-lock-held-until-spawn"); + paths.ensure()?; + let lock_path = paths.managed_launch_lock_path(); + let lock_was_held = std::rc::Rc::new(std::cell::Cell::new(false)); + let observed = lock_was_held.clone(); + TEST_SPAWN_HOOK.with(|hook| { + *hook.borrow_mut() = Some(std::rc::Rc::new(move |_command| { + observed.set( + rocm_core::FileLock::acquire_bounded(&lock_path, Duration::ZERO).is_err(), + ); + Err(io::Error::other("spawn intercepted by test")) + })); + }); + let launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; + let result = if attached { + run_attached_service( + &paths, + "lemonade", + "lemonade-lock-attached", + "qwen", + &resolve, + "127.0.0.1", + serve_port::PortRequest::Auto, + &[], + None, + None, + None, + launch_lock, + false, + ) + } else { + start_managed_service( + &paths, + "lemonade", + "lemonade-lock-start", + "qwen", + &resolve, + "127.0.0.1", + serve_port::PortRequest::Auto, + &resolve.device_policy, + &[], + None, + None, + None, + None, + launch_lock, + false, + &mut |_elapsed| {}, + ) + .map(|_| ()) + }; + TEST_SPAWN_HOOK.with(|hook| *hook.borrow_mut() = None); + let _ = fs::remove_dir_all(root); + assert!( + format!("{:#}", result.expect_err("the intercepted spawn fails")) + .contains("spawn intercepted by test") + ); + assert!( + lock_was_held.get(), + "the launch lock must be held at spawn (attached: {attached})" + ); + } + Ok(()) + } + + #[test] + fn resolve_endpoint_auth_loopback_stays_credential_free() { + // Loopback binds never require auth, even if a key is supplied. + for host in ["127.0.0.1", "localhost", "::1"] { + assert_eq!(resolve_endpoint_auth(host, None, false).unwrap(), None); + assert_eq!( + resolve_endpoint_auth(host, Some("ignored"), false).unwrap(), + None + ); + } + } + + #[test] + fn resolve_endpoint_auth_loopback_can_be_required_when_something_republishes_it() { + // "Loopback" describes the bind address, not who can reach the port. A + // tailnet publish, a proxy, or a container port map all leave the bind + // loopback while widening the audience, and the default policy would + // hand out an unauthenticated endpoint. Whoever widens the reach asks + // for the credential explicitly. + for host in ["127.0.0.1", "localhost", "::1"] { + let generated = resolve_endpoint_auth(host, None, true) + .unwrap() + .expect("a required key must be generated, not skipped"); + assert!(!generated.trim().is_empty()); + + assert_eq!( + resolve_endpoint_auth(host, Some("supplied-key"), true).unwrap(), + Some("supplied-key".to_owned()), + "a supplied key must be honoured rather than ignored as it is by default" + ); + } + + // The same validation a public bind gets: an empty key is a refusal, not + // a silent downgrade to no auth. + assert!(resolve_endpoint_auth("127.0.0.1", Some(" "), true).is_err()); + } + + #[test] + fn resolve_endpoint_auth_public_uses_supplied_key_trimmed() { + let key = resolve_endpoint_auth("0.0.0.0", Some(" my-key "), false) + .unwrap() + .expect("public bind must have a key"); + assert_eq!(key, "my-key"); + } + + #[test] + fn resolve_endpoint_auth_public_generates_key_when_absent() { let key = resolve_endpoint_auth("0.0.0.0", None, false) .unwrap() .expect("public bind must generate a key"); @@ -1764,4 +2571,783 @@ mod tests { assert_eq!(read_autostart_claim(&path), None); let _ = fs::remove_dir_all(&root); } + + #[test] + fn duplicate_managed_launch_detected_across_distinct_service_ids() -> Result<()> { + // `generate_service_id` embeds a timestamp, so a second launch for the + // same engine+model has a DIFFERENT service_id. The guard must still + // detect the live service by (engine, canonical_model_id), and return + // the newest live match. `starting` skips the endpoint probe; the + // current process id is a guaranteed-live PID. + let (root, paths) = test_paths("dup-managed-distinct-ids"); + paths.ensure()?; + + // Older, dead manifest for the same engine+model (distinct service_id). + let mut dead = ManagedServiceRecord::new( + &paths, + "lemonade-qwen-1000", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + 11500, + "managed", + 999_999_999, + None, + None, + None, + ); + dead.status = "ready".to_owned(); + dead.engine_pid = Some(999_999_999); + dead.created_at_unix_ms = 1000; + dead.write()?; + + // Newer, live manifest for the same engine+model (distinct service_id). + let mut live = ManagedServiceRecord::new( + &paths, + "lemonade-qwen-2000", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + 11501, + "managed", + std::process::id(), + None, + None, + None, + ); + live.status = "starting".to_owned(); + live.engine_pid = Some(std::process::id()); + live.created_at_unix_ms = 2000; + live.write()?; + + let live_services = live_managed_services(&paths); + let found = + existing_live_managed_service_in(&live_services, "lemonade", "qwen-canonical").cloned(); + let _ = fs::remove_dir_all(root); + + let found = found.expect("a live managed service should be detected by engine+model"); + assert_eq!( + found.service_id, "lemonade-qwen-2000", + "should return the newest live match" + ); + assert!(managed_service_is_live(&found)); + Ok(()) + } + + #[test] + fn dead_managed_service_allows_relaunch() -> Result<()> { + // A stale manifest with dead PIDs must NOT block a relaunch: liveness + // refresh demotes it to "stopped", so the guard returns None. + let (root, paths) = test_paths("dup-managed-dead"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( + &paths, + "lemonade-qwen-3000", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + 11502, + "managed", + 999_999_999, + None, + None, + None, + ); + record.status = "ready".to_owned(); + record.engine_pid = Some(999_999_999); + record.write()?; + + let live = live_managed_services(&paths); + let found = existing_live_managed_service_in(&live, "lemonade", "qwen-canonical"); + let _ = fs::remove_dir_all(root); + + assert!( + found.is_none(), + "a dead managed service must not block relaunch" + ); + Ok(()) + } + + #[test] + fn live_service_for_other_model_does_not_block() -> Result<()> { + // A live service for a DIFFERENT model must not match — the guard keys + // on the model, not just the engine. + let (root, paths) = test_paths("dup-managed-other-model"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( + &paths, + "lemonade-other-1", + "lemonade", + "other", + "other-canonical", + "127.0.0.1", + 11503, + "managed", + std::process::id(), + None, + None, + None, + ); + record.status = "starting".to_owned(); + record.engine_pid = Some(std::process::id()); + record.write()?; + + let live = live_managed_services(&paths); + let found = existing_live_managed_service_in(&live, "lemonade", "qwen-canonical"); + let _ = fs::remove_dir_all(root); + + assert!( + found.is_none(), + "a live service for a different model must not match" + ); + Ok(()) + } + + #[test] + fn missing_manifest_allows_launch() { + // No services dir / manifests → nothing to detect, launch proceeds. + let (root, paths) = test_paths("dup-managed-missing"); + let live = live_managed_services(&paths); + let found = existing_live_managed_service_in(&live, "lemonade", "qwen-canonical"); + let _ = fs::remove_dir_all(root); + assert!(found.is_none()); + } + + fn write_live_service_record( + paths: &AppPaths, + service_id: &str, + canonical_model_id: &str, + port: u16, + status: &str, + ) -> Result { + let mut record = ManagedServiceRecord::new( + paths, + service_id, + "lemonade", + canonical_model_id, + canonical_model_id.to_owned(), + rocm_core::DEFAULT_LOCAL_HOST, + port, + "managed", + std::process::id(), + None, + None, + None, + ); + record.status = status.to_owned(); + record.engine_pid = Some(std::process::id()); + record.write()?; + Ok(record) + } + + fn never_probe(_port: u16) -> std::io::Result { + panic!("no port may be probed on this path") + } + + fn always_free(port: u16) -> std::io::Result { + Ok(rocm_core::LoopbackPortLease::engine_owned(port)) + } + + #[test] + fn allocation_transaction_answers_an_equivalent_live_service_before_leasing() -> Result<()> { + let (root, paths) = test_paths("alloc-txn-already-live"); + paths.ensure()?; + let existing = write_live_service_record( + &paths, + "lemonade-qwen-9000", + "qwen-canonical", + 11501, + "starting", + )?; + + let decision = resolve_managed_port_in_transaction( + &paths, + "lemonade", + "qwen-canonical", + "127.0.0.1", + serve_port::PortRequest::Auto, + &mut never_probe, + )?; + let _ = fs::remove_dir_all(root); + + match decision { + ManagedPortDecision::AlreadyLive(record) => { + assert_eq!(record.service_id, existing.service_id); + assert_eq!(record.port, existing.port); + } + ManagedPortDecision::Lease(_) => { + panic!("an equivalent live service must be answered, not leased a new port") + } + } + Ok(()) + } + + #[test] + fn allocation_transaction_skips_ports_reserved_by_other_live_services() -> Result<()> { + let (root, paths) = test_paths("alloc-txn-skips-reserved"); + paths.ensure()?; + write_live_service_record(&paths, "lemonade-a", "model-a", 11_435, "running")?; + write_live_service_record(&paths, "lemonade-b", "model-b", 11_436, "recovering")?; + + let decision = resolve_managed_port_in_transaction( + &paths, + "lemonade", + "model-c", + "127.0.0.1", + serve_port::PortRequest::Auto, + &mut always_free, + )?; + let _ = fs::remove_dir_all(root); + + match decision { + ManagedPortDecision::Lease(lease) => assert_eq!(lease.port(), 11_437), + ManagedPortDecision::AlreadyLive(record) => { + panic!("model-c has no live service, expected a lease, got {record:?}") + } + } + Ok(()) + } + + #[test] + fn allocation_transaction_ignores_stopped_records() -> Result<()> { + let (root, paths) = test_paths("alloc-txn-ignores-stopped"); + paths.ensure()?; + let mut stale = + write_live_service_record(&paths, "lemonade-stale", "model-stale", 11_435, "running")?; + // A dead pid demotes the record to stopped when liveness is refreshed, so + // it must not reserve its port. + stale.engine_pid = Some(999_999_999); + stale.supervisor_pid = 999_999_999; + stale.write()?; + + let decision = resolve_managed_port_in_transaction( + &paths, + "lemonade", + "model-fresh", + "127.0.0.1", + serve_port::PortRequest::Auto, + &mut always_free, + )?; + let _ = fs::remove_dir_all(root); + + match decision { + ManagedPortDecision::Lease(lease) => assert_eq!(lease.port(), 11_435), + ManagedPortDecision::AlreadyLive(record) => { + panic!("a stopped record must not block a fresh launch, got {record:?}") + } + } + Ok(()) + } + + #[test] + fn stopping_the_reserving_record_frees_its_explicit_port_for_reuse() -> Result<()> { + // Proves the mechanism `rocm services stop` relies on: it never touches + // the port directly, it only flips the stopped record's `status` (see + // the `all_stopped` branch in `main.rs`'s stop handler). This recreates + // that exact disk state and checks the same explicit port a live + // competitor refused now resolves, closing the loop the E2E + // `assert_port_reserved_message` step only checks by wording. + let (root, paths) = test_paths("alloc-explicit-port-freed-by-stop"); + paths.ensure()?; + let mut holder = ManagedServiceRecord::new( + &paths, + "lemonade-a-1", + "lemonade", + "model-a", + "model-a".to_owned(), + DEFAULT_LOCAL_HOST, + 11_435, + "managed", + std::process::id(), + None, + None, + None, + ); + holder.status = "running".to_owned(); + holder.engine_pid = Some(std::process::id()); + holder.write()?; + + let still_reserved = resolve_managed_port_in_transaction( + &paths, + "lemonade", + "model-b", + DEFAULT_LOCAL_HOST, + serve_port::PortRequest::Explicit(11_435), + &mut always_free, + ); + // The refusal names the service AND the command that frees the port; + // the stop below is that command's own implementation, and the grant + // after it is the outcome, so advice and result are pinned together. + let stop_hint = service_hints::service_stop_hint("lemonade-a-1"); + assert!( + matches!(still_reserved, Err(ref error) if error.to_string().contains("already reserved by managed service lemonade-a-1") + && error.to_string().contains(&stop_hint)), + "unexpected result while still running: {still_reserved:?}" + ); + + // Go through the real stop path (`rocm services stop --yes`'s own + // implementation) rather than writing `status = "stopped"` to disk + // directly, so this test actually proves the advice serve-29 pins: that + // running the stop command frees the port. The termination closure is + // the same seam production injects a real signal through; here it just + // confirms every recorded process gone without touching the test + // process itself. + crate::services_text::stop_internal_managed_service_with(&paths, "lemonade-a-1", |_| { + (Vec::new(), true) + })?; + + let decision = resolve_managed_port_in_transaction( + &paths, + "lemonade", + "model-b", + DEFAULT_LOCAL_HOST, + serve_port::PortRequest::Explicit(11_435), + &mut always_free, + ); + let _ = fs::remove_dir_all(root); + + match decision? { + ManagedPortDecision::Lease(lease) => assert_eq!(lease.port(), 11_435), + ManagedPortDecision::AlreadyLive(existing) => { + panic!( + "a stopped record must not block reuse: {}", + existing.service_id + ) + } + } + Ok(()) + } + + #[test] + fn allocation_transaction_rejects_an_explicit_port_a_recovering_service_holds() -> Result<()> { + let (root, paths) = test_paths("alloc-txn-rejects-explicit-collision"); + paths.ensure()?; + let existing = write_live_service_record( + &paths, + "lemonade-recovering", + "model-recovering", + 11_435, + "recovering", + )?; + + let result = resolve_managed_port_in_transaction( + &paths, + "lemonade", + "model-other", + "127.0.0.1", + serve_port::PortRequest::Explicit(11_435), + &mut never_probe, + ); + let _ = fs::remove_dir_all(root); + + let Err(error) = result else { + panic!("an explicit port a recovering service holds must be refused") + }; + let message = error.to_string(); + assert!( + message.contains(&existing.service_id), + "the refusal must name the service holding the port: {message}" + ); + assert!( + message.contains("recovering"), + "the refusal must say why the port is unavailable: {message}" + ); + Ok(()) + } + + #[test] + fn concurrent_auto_transactions_publish_distinct_ports() -> Result<()> { + let (root, paths) = test_paths("alloc-txn-concurrent-distinct"); + paths.ensure()?; + let paths = std::sync::Arc::new(paths); + + let spawn_one = |model: &'static str| { + let paths = std::sync::Arc::clone(&paths); + thread::spawn(move || -> Result { + let _launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; + let lease = match resolve_managed_port_in_transaction( + &paths, + "lemonade", + model, + "127.0.0.1", + serve_port::PortRequest::Auto, + &mut serve_port::loopback_bind_probe, + )? { + ManagedPortDecision::Lease(lease) => lease, + ManagedPortDecision::AlreadyLive(record) => { + panic!("fresh model must not already be live: {record:?}") + } + }; + let port = lease.port(); + write_live_service_record(&paths, model, model, port, "running")?; + lease.release(); + Ok(port) + }) + }; + + let first = spawn_one("model-one"); + let second = spawn_one("model-two"); + let first_port = first.join().expect("thread one panicked")?; + let second_port = second.join().expect("thread two panicked")?; + let _ = fs::remove_dir_all(root); + + assert_ne!(first_port, second_port); + for port in [first_port, second_port] { + assert!( + (serve_port::AUTO_PORT_FIRST..=serve_port::AUTO_PORT_LAST).contains(&port), + "leased port {port} must fall within the auto-select range" + ); + } + Ok(()) + } + + #[test] + fn concurrent_equivalent_transactions_converge_on_one_record() -> Result<()> { + let (root, paths) = test_paths("alloc-txn-concurrent-converge"); + paths.ensure()?; + let paths = std::sync::Arc::new(paths); + + let race_once = || { + let paths = std::sync::Arc::clone(&paths); + thread::spawn(move || -> Result { + let _launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; + match resolve_managed_port_in_transaction( + &paths, + "lemonade", + "shared-model", + "127.0.0.1", + serve_port::PortRequest::Auto, + &mut serve_port::loopback_bind_probe, + )? { + ManagedPortDecision::Lease(lease) => { + let port = lease.port(); + write_live_service_record( + &paths, + "lemonade-shared", + "shared-model", + port, + "running", + )?; + lease.release(); + Ok(true) + } + ManagedPortDecision::AlreadyLive(_) => Ok(false), + } + }) + }; + + let first = race_once(); + let second = race_once(); + let first_launched = first.join().expect("thread one panicked")?; + let second_launched = second.join().expect("thread two panicked")?; + + let live_count = live_managed_services(&paths).len(); + let _ = fs::remove_dir_all(root); + + assert_ne!( + first_launched, second_launched, + "exactly one racing transaction must launch" + ); + assert_eq!(live_count, 1, "only one record may end up live"); + Ok(()) + } + + #[test] + fn select_gpu_indices_under_launch_lock_blocks_until_the_holder_releases() -> Result<()> { + // Exercises the real lock-acquisition function production callers use + // (`select_gpu_indices_under_launch_lock`) against a lock another thread + // actually holds, so a refactor that dropped the lock early would fail + // here. `spawn_managed_engine_child` is then run for real, but only via + // its `AlreadyLive` short-circuit (a pre-seeded duplicate record), so no + // child process is ever launched. The holder writes nothing: this does + // not show a write made under the lock being observed. + use std::sync::{Arc, Barrier}; + use std::time::Instant; + + let (root, paths) = test_paths("launch-lock-holds-across-spawn"); + paths.ensure()?; + write_live_service_record(&paths, "lemonade-held-1", "held-canonical", 11_520, "ready")?; + + let barrier = Arc::new(Barrier::new(2)); + let hold_for = Duration::from_millis(250); + + let holder_paths = paths.clone(); + let holder_barrier = Arc::clone(&barrier); + let holder = thread::spawn(move || -> Result<()> { + let _lock = rocm_core::FileLock::acquire(holder_paths.managed_launch_lock_path())?; + holder_barrier.wait(); + thread::sleep(hold_for); + Ok(()) + }); + + barrier.wait(); + let started = Instant::now(); + let (gpu_indices, launch_lock) = crate::gpu_policy::select_gpu_indices_under_launch_lock( + &paths, + true, + None, + || None, + None, + None, + )?; + let waited = started.elapsed(); + + let resolve = ResolveModelResponse { + canonical_model_id: "held-canonical".to_owned(), + served_model_id: None, + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + let spawn = spawn_managed_engine_child( + &paths, + "lemonade", + "lemonade-held-2", + "held-model", + &resolve, + DEFAULT_LOCAL_HOST, + serve_port::PortRequest::Auto, + &resolve.device_policy, + &gpu_indices, + None, + None, + None, + false, + ); + drop(launch_lock); + holder.join().expect("holder thread")?; + let _ = fs::remove_dir_all(root); + + // `thread::sleep` only guarantees *at least* `hold_for`, but under a + // loaded host (e.g. a full parallel `cargo test` run on WSL2) timer + // coalescing can make the measured wait a hair short of the nominal + // hold. Allow a small margin so the test still fails loudly if the + // call returns early, without being flaky on actual block duration. + let margin = Duration::from_millis(20); + assert!( + waited + margin >= hold_for, + "select_gpu_indices_under_launch_lock returned before the holder released the lock: waited {waited:?}, held {hold_for:?}" + ); + match spawn? { + ManagedSpawn::AlreadyRunning(report) => { + assert_eq!(report.service_id, "lemonade-held-1"); + } + ManagedSpawn::Spawned { .. } => { + panic!( + "expected the pre-seeded AlreadyLive record to short-circuit, not a real spawn" + ) + } + } + Ok(()) + } + + #[test] + fn abandoned_launch_leaves_no_record_log_or_endpoint_key() -> Result<()> { + let (root, paths) = test_paths("abandon-launch-cleanup"); + paths.ensure()?; + let mut record = write_live_service_record( + &paths, + "lemonade-abandon", + "abandon-model", + 11_490, + "running", + )?; + fs::create_dir_all( + record + .engine_state_path + .parent() + .expect("engine state parent"), + )?; + fs::write(&record.engine_state_path, "{}")?; + endpoint_keys::store_endpoint_api_key(&paths, &record.service_id, "secret-key")?; + // Simulate a pre-spawn failure: nothing ever ran. + record.engine_pid = None; + + abandon_failed_managed_launch(&paths, &record); + + assert!(!record.manifest_path.exists()); + assert!(!record.log_path.exists()); + assert!(!record.engine_state_path.exists()); + assert_eq!( + endpoint_keys::endpoint_api_key(&paths, &record.service_id), + None + ); + assert!(live_managed_services(&paths).is_empty()); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + #[cfg(not(windows))] + #[ignore = "spawned as a child process by abandoned_launch_terminates_the_spawned_child"] + fn managed_launch_cleanup_sleeper_child() { + if std::env::var_os("ROCM_TEST_SLEEPER_CHILD").is_some() { + thread::sleep(Duration::from_mins(2)); + } + } + + #[test] + #[cfg(not(windows))] + fn abandoned_launch_terminates_the_spawned_child() -> Result<()> { + let (root, paths) = test_paths("abandon-launch-terminates-child"); + paths.ensure()?; + let mut record = write_live_service_record( + &paths, + "lemonade-abandon-child", + "abandon-child-model", + 11_491, + "running", + )?; + + let current_exe = std::env::current_exe().expect("current test exe"); + let mut child = ProcessCommand::new(current_exe) + .arg("--exact") + .arg("service_launch::tests::managed_launch_cleanup_sleeper_child") + .arg("--ignored") + .arg("--test-threads=1") + .env("ROCM_TEST_SLEEPER_CHILD", "1") + .spawn() + .expect("spawn sleeper child"); + let child_pid = child.id(); + record.engine_pid = Some(child_pid); + + abandon_failed_managed_launch(&paths, &record); + + let wait_result = child.wait(); + + match wait_result { + Ok(status) => assert!( + !status.success(), + "the sleeper child must be terminated, not exit cleanly" + ), + Err(error) => panic!("failed to wait for sleeper child: {error}"), + } + let persisted = load_managed_services(&paths) + .expect("re-read manifests") + .into_iter() + .find(|candidate| candidate.service_id == record.service_id); + let _ = fs::remove_dir_all(root); + let persisted = persisted.expect( + "a launch that spawned a child must keep its record so `rocm services`/`rocm logs` can show the attempt", + ); + assert_eq!( + persisted.status, "failed", + "a spawned child's record must be marked failed, not deleted" + ); + Ok(()) + } + + #[test] + #[cfg(not(windows))] + fn launch_releases_the_leased_port_before_spawning_the_child() -> Result<()> { + // `lease.release()` must run before the child is spawned, not after, + // or the engine is handed a port the CLI is still holding open. + // Proven via the injected `spawn_override` seam: it asserts a fresh + // bind on the leased port succeeds at the exact instant production + // would spawn, which only holds if the lease was already released. + let (root, paths) = test_paths("launch-releases-port-before-spawn"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + // Ephemeral, not a fixed number in the automatic range, which a real + // `rocm serve` on the developer's machine could be holding. + let port = std::net::TcpListener::bind(("127.0.0.1", 0))? + .local_addr()? + .port(); + let lease = rocm_core::lease_loopback_port(port)?; + let mut record = ManagedServiceRecord::new( + &paths, + "lemonade-release-before-spawn", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + port, + "managed", + 0, + None, + None, + None, + ); + record.write()?; + + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + served_model_id: None, + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + + let bind_succeeded = std::cell::Cell::new(false); + let spawn_override: &dyn Fn(&mut ProcessCommand) -> io::Result = + &|_command| { + // Retry briefly: a sibling test's concurrent fork can hold a copy + // of the just-released socket until its exec closes it. + let deadline = std::time::Instant::now() + Duration::from_secs(2); + while !bind_succeeded.get() && std::time::Instant::now() < deadline { + bind_succeeded.set(std::net::TcpListener::bind(("127.0.0.1", port)).is_ok()); + } + // A long-lived no-op child stands in for the engine: `_command` + // carries real serve args this test binary cannot execute as + // itself (see `managed_launch_cleanup_sleeper_child`). + ProcessCommand::new(std::env::current_exe().expect("test binary")) + .arg("service_launch::tests::managed_launch_cleanup_sleeper_child") + .arg("--exact") + .arg("--ignored") + .env("ROCM_TEST_SLEEPER_CHILD", "1") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + }; + + let result = launch_managed_engine_child_locked( + &paths, + &mut record, + lease, + "lemonade", + "lemonade-release-before-spawn", + &resolve, + "127.0.0.1", + &resolve.device_policy, + &[], + None, + None, + None, + Some(spawn_override), + ); + if let Ok(child_pid) = result.as_ref() { + let _ = rocm_core::terminate_process(*child_pid); + } + let _ = fs::remove_dir_all(root); + + assert!( + bind_succeeded.get(), + "the port must already be bindable (released) by the time the child is spawned" + ); + result?; + Ok(()) + } } diff --git a/apps/rocm/src/services_text.rs b/apps/rocm/src/services_text.rs index 9626aecf3..477474518 100644 --- a/apps/rocm/src/services_text.rs +++ b/apps/rocm/src/services_text.rs @@ -754,7 +754,10 @@ fn stop_internal_managed_service(paths: &AppPaths, service_id: &str) -> Result (Vec, bool), @@ -975,12 +978,47 @@ pub(crate) fn restart_internal_managed_service( /// [`restart_internal_managed_service`], with the termination of the recorded /// processes supplied by the caller — see [`stop_internal_managed_service_with`]. +/// +/// The respawn is serialized on the same cross-process launch lock a fresh +/// `rocm serve` launch and `rocmd` recovery both use: the confirmed stop just +/// released this service's (host, port), and nothing reserves it again until +/// the new child binds and its record is published, so a concurrent launch or +/// recovery could otherwise claim the exact port this restart is about to +/// reuse. fn restart_internal_managed_service_with( paths: &AppPaths, service_id: &str, runtime_pin: Option<&str>, terminate: impl FnOnce(&ManagedServiceRecord) -> (Vec, bool), ) -> Result { + restart_internal_managed_service_inner(paths, service_id, runtime_pin, terminate, None) +} + +/// Replaces the child `spawn()` of a restart (non-Windows only; Windows spawns +/// through `rocm_core::spawn_detached_no_inherit`). Test-only seam, so a test +/// can observe the leased port at the exact instant production would spawn. +/// How long a restart waits for the new child to serve. +#[allow(clippy::missing_const_for_fn)] // not const under `cfg(test)`, where a test shortens it +fn restart_readiness_timeout() -> Duration { + #[cfg(test)] + if let Some(timeout) = tests::READINESS_TIMEOUT_OVERRIDE.with(std::cell::Cell::get) { + return timeout; + } + Duration::from_secs(45) +} + +type RestartSpawnOverride<'a> = + dyn Fn(&mut std::process::Command) -> io::Result + 'a; + +fn restart_internal_managed_service_inner( + paths: &AppPaths, + service_id: &str, + runtime_pin: Option<&str>, + terminate: impl FnOnce(&ManagedServiceRecord) -> (Vec, bool), + spawn_override: Option<&RestartSpawnOverride<'_>>, +) -> Result { + #[cfg(windows)] + let _ = spawn_override; let record = load_service_for_restart(paths, service_id, runtime_pin)?; // Preserve the endpoint key across the restart: `stop_internal_managed_service` // deletes the key file (correct on a real stop), but a restart must bring the @@ -1024,8 +1062,45 @@ fn restart_internal_managed_service_with( let (stopped_record, _) = read_managed_service_manifest(paths, service_id)?; // Still after the stop, like the device-policy refusal inside it always was: // the callers' "stopped by the attempt" report and pin restore both cover a - // restart that gets this far and fails. + // restart that gets this far and fails. Pure computation — it parses the + // record, it does not touch the network — so it stays ahead of the launch + // lock below and a refused device policy still fails before anything binds. let serve_args = restart_child_argv(&record)?; + // The confirmed stop just released this (host, port): nothing holds it again + // until the new child binds and `record_restarted_child` publishes below. + // Close that gap the same way a fresh launch and `rocmd` recovery do — + // under the shared launch lock, with a re-check against the disk and a + // loopback preflight — so a concurrent launch or recovery can't claim this + // exact port out from under the restart. Unbounded, like every other + // CLI-side acquire of this lock (see `FileLock::acquire_bounded`'s doc + // comment for why only `rocmd`'s recovery bounds its wait). + let launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; + if let Some(collision) = load_managed_services(paths) + .context("failed to re-read managed services under the launch lock")? + .into_iter() + .find(|other| { + other.service_id != stopped_record.service_id && other.blocks_port(stopped_record.port) + }) + { + bail!( + "cannot restart `{service_id}` on {}:{}: managed service {} is recorded on that \ + port (it may have claimed that port while the restart waited for the launch lock)", + stopped_record.host, + stopped_record.port, + collision.service_id + ); + } + let port_lease = if rocm_core::is_canonical_loopback_host(&stopped_record.host) { + Some(rocm_core::lease_loopback_port_or_bind_error( + stopped_record.port, + format_args!( + "cannot restart `{service_id}` on {}:{}: its port is in use", + stopped_record.host, stopped_record.port + ), + )?) + } else { + None + }; fs::OpenOptions::new() .create(true) .append(true) @@ -1043,6 +1118,11 @@ fn restart_internal_managed_service_with( record.runtime_id.as_deref(), record.env_id.as_deref(), )?; + // Hand the port to the respawned child: close the probe socket immediately + // before the spawn, still holding the launch lock. + if let Some(lease) = port_lease { + lease.release(); + } // A restarted public service comes back authenticated: its 0600 key file // persists across restarts (removed only on stop), so hand the child the same // file and reuse the key for the readiness probe. @@ -1071,9 +1151,11 @@ fn restart_internal_managed_service_with( if let Some(key_file) = endpoint_key_file.as_deref() { command.env(rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV, key_file); } - let mut child = command - .spawn() - .context("failed to restart managed engine process")?; + let mut child = match spawn_override { + Some(spawn) => spawn(&mut command), + None => command.spawn(), + } + .context("failed to restart managed engine process")?; thread::sleep(Duration::from_millis(200)); if let Some(status) = child .try_wait() @@ -1084,7 +1166,10 @@ fn restart_internal_managed_service_with( })?; bail!( "{}", - managed_engine_startup_failure_detail(status, &record.log_path) + managed_engine_startup_failure_detail( + &format!(" with status {status}"), + &record.log_path + ) ); } child.id() @@ -1094,13 +1179,17 @@ fn restart_internal_managed_service_with( // Recorded before the readiness wait, so a stop issued during it finds the // new child rather than the processes already stopped. let started = record_restarted_child(paths, &stopped_record, runtime_pin, child_pid)?; + // Published; a concurrent launch or recovery now sees this service as live + // on its port again, so there is nothing left for the lock to protect, and + // the readiness wait below can block for many seconds. + drop(launch_lock); let readiness = wait_for_service_http_ready( &started.engine, &started.host, started.port, started.served_model_name(), endpoint_api_key.as_deref(), - Duration::from_secs(45), + restart_readiness_timeout(), ); record_restart_readiness(paths, &started, readiness) } @@ -2483,6 +2572,361 @@ mod tests { assert_eq!(on_disk.restart_count, 0, "nothing was written"); } + /// The gap a manual restart used to leave open: the confirmed stop just + /// released the service's port, and nothing reserves it again until the + /// new child binds and `record_restarted_child` publishes — a concurrent + /// `rocm serve` launch could claim that exact port in between. Recreate + /// the race under the shared launch lock and require the restart to + /// refuse rather than spawn a competitor onto it, following the same + /// wait-then-publish-then-release shape as + /// `services_prune_waits_for_the_managed_launch_lock_before_sweeping`. + #[test] + fn restart_refuses_a_port_a_launch_claimed_while_it_waited_for_the_lock() -> Result<()> { + use std::sync::mpsc; + + const WAIT_PROOF: Duration = Duration::from_millis(500); + + let (root, paths) = test_paths("restart-launch-claimed-port"); + paths.ensure()?; + let service_id = "svc-restart-launch-claimed-port"; + let record = stop_test_record(&paths, service_id); + let host = record.host.clone(); + let port = record.port; + record.write()?; + + let launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; + + let (started_tx, started_rx) = mpsc::channel(); + let (done_tx, done_rx) = mpsc::channel(); + let (waited, finished) = std::thread::scope(|scope| { + scope.spawn(|| { + started_tx.send(()).expect("signal about to restart"); + let outcome = + restart_internal_managed_service_with(&paths, service_id, None, |_| { + (Vec::new(), true) + }); + let _ = done_tx.send(outcome.map_err(|error| format!("{error:#}"))); + }); + started_rx + .recv_timeout(Duration::from_secs(10)) + .expect("restart thread started"); + let early = done_rx.recv_timeout(WAIT_PROOF); + let waited = early.is_err(); + // Simulate a fresh `rocm serve --gpu auto` publishing a brand-new + // service on this exact port while the restart was still waiting + // for the lock it itself now holds. + let mut competitor = ManagedServiceRecord::new( + &paths, + "svc-fresh-launch", + "lemonade", + "other-model", + "other-model-canonical", + &host, + port, + "managed", + 0, + None, + None, + None, + ); + competitor.status = "starting".to_owned(); + competitor.write().expect("publish the competing record"); + drop(launch_lock); + let finished = match early { + Ok(finished) => finished, + Err(_) => done_rx + .recv_timeout(Duration::from_secs(10)) + .expect("restart completes once the launch lock is released"), + }; + (waited, finished) + }); + let _ = fs::remove_dir_all(root); + + assert!( + waited, + "the restart must serialize on the shared launch lock" + ); + let error = + finished.expect_err("a live competitor on the same port must refuse the restart"); + assert!(error.contains("svc-fresh-launch"), "{error}"); + assert!(error.contains("claimed that port"), "{error}"); + Ok(()) + } + + /// The disk re-check above only catches a recorded competitor. Something + /// bound to the exact port without ever publishing a manifest for it — a + /// stray process, a one-off `nc -l` — would sail through that check; only + /// the loopback lease actually binding the port catches it. A real + /// listener held on the port is what proves that lease ran, not just a + /// record on disk. + #[test] + fn restart_refuses_a_port_something_is_actually_bound_to() -> Result<()> { + let (root, paths) = test_paths("restart-port-actually-bound"); + paths.ensure()?; + let service_id = "svc-restart-port-actually-bound"; + let record = stop_test_record(&paths, service_id); + let port = record.port; + record.write()?; + + let _blocker = std::net::TcpListener::bind(("127.0.0.1", port))?; + + let result = + restart_internal_managed_service_with(&paths, service_id, None, |_| (Vec::new(), true)); + + let _ = fs::remove_dir_all(root); + let error = + result.expect_err("a port actually bound by someone else must refuse the restart"); + let rendered = format!("{error:#}"); + assert!(rendered.contains("its port is in use"), "{rendered}"); + Ok(()) + } + + /// Mirrors `launch_releases_the_leased_port_before_spawning_the_child` and + /// its rocmd twin: the restart's probe lease must be closed by the time the + /// child is spawned, or the child's own bind of the port fails. Without + /// the `lease.release()` call every loopback restart would die at the + /// engine bind, and nothing else here would notice. + #[cfg(not(windows))] + #[test] + fn restart_releases_the_leased_port_before_spawning_the_child() -> Result<()> { + let (root, paths) = test_paths("restart-releases-lease"); + paths.ensure()?; + let service_id = "svc-restart-releases-lease"; + let record = stop_test_record(&paths, service_id); + let port = record.port; + record.write()?; + + let bind_succeeded = std::cell::Cell::new(false); + let lock_still_held = std::cell::Cell::new(false); + let lock_path = paths.managed_launch_lock_path(); + let spawn: &RestartSpawnOverride<'_> = &|_command| { + // The launch lock must outlive the spawn: it is what keeps a + // concurrent launch from claiming the port before this record is + // republished. + lock_still_held + .set(rocm_core::FileLock::acquire_bounded(&lock_path, Duration::ZERO).is_err()); + // Retry briefly: a sibling test's concurrent fork can hold a copy of + // the just-released socket until its exec closes it. + let deadline = std::time::Instant::now() + Duration::from_secs(2); + while !bind_succeeded.get() && std::time::Instant::now() < deadline { + bind_succeeded.set(std::net::TcpListener::bind(("127.0.0.1", port)).is_ok()); + } + // Fail the spawn on purpose: the observation above is all this + // test needs, and no real engine should start. + Err(io::Error::other("spawn intercepted by test")) + }; + let result = restart_internal_managed_service_inner( + &paths, + service_id, + None, + |_| (Vec::new(), true), + Some(spawn), + ); + let _ = fs::remove_dir_all(root); + + assert!( + bind_succeeded.get(), + "the leased port must already be bindable (released) by the time the child is spawned" + ); + assert!( + lock_still_held.get(), + "the launch lock must still be held when the restart spawns" + ); + let rendered = format!("{:#}", result.expect_err("the intercepted spawn must fail")); + assert!( + rendered.contains("spawn intercepted by test"), + "the restart must have reached the spawn: {rendered}" + ); + Ok(()) + } + + #[cfg(not(windows))] + #[test] + fn restart_is_not_blocked_by_a_stopped_or_failed_record_on_the_same_port() -> Result<()> { + // Mirror of `restart_refuses_a_port_a_launch_claimed...`: `blocks_port` is + // `is_live() && port == port`, so a `stopped`/`failed` sibling on the same + // port must not trip the "claimed that port" refusal. Mutation-proof: if + // `blocks_port` regressed to ignore status, the restart would bail there + // instead of reaching the real (and, in this test harness, doomed) child + // spawn below. + let (root, paths) = test_paths("restart-non-live-same-port"); + paths.ensure()?; + let service_id = "svc-restart-non-live-same-port"; + let record = stop_test_record(&paths, service_id); + let host = record.host.clone(); + let port = record.port; + record.write()?; + + let mut stopped_sibling = ManagedServiceRecord::new( + &paths, + "svc-stopped-sibling", + "lemonade", + "other-model", + "other-model-canonical", + &host, + port, + "managed", + 0, + None, + None, + None, + ); + stopped_sibling.status = "stopped".to_owned(); + stopped_sibling.write()?; + + let mut failed_sibling = ManagedServiceRecord::new( + &paths, + "svc-failed-sibling", + "lemonade", + "other-model-2", + "other-model-2-canonical", + &host, + port, + "managed", + 0, + None, + None, + None, + ); + failed_sibling.status = "failed".to_owned(); + failed_sibling.write()?; + + // Fail the spawn on purpose: reaching it is what proves neither sibling + // blocked the restart, so any other error is a real failure here. + let spawn: &RestartSpawnOverride<'_> = + &|_command| Err(io::Error::other("spawn intercepted by test")); + let result = restart_internal_managed_service_inner( + &paths, + service_id, + None, + |_| (Vec::new(), true), + Some(spawn), + ); + + let _ = fs::remove_dir_all(root); + let rendered = format!("{:#}", result.expect_err("the intercepted spawn must fail")); + assert!( + rendered.contains("spawn intercepted by test"), + "a stopped or failed sibling on the same port must not block the restart: {rendered}" + ); + Ok(()) + } + + thread_local! { + /// Lets a test that lets a restart reach its readiness wait shorten it. + pub(super) static READINESS_TIMEOUT_OVERRIDE: std::cell::Cell> = + const { std::cell::Cell::new(None) }; + } + + /// The restart holds the launch lock until its record names the new child. + /// A waiter blocked on the lock since the spawn must therefore read a record + /// that already carries the child's pid the moment it gets in; releasing + /// the lock right after the spawn lets it read the stale record instead. + #[cfg(not(windows))] + #[test] + fn restart_holds_the_launch_lock_until_the_record_names_the_new_child() -> Result<()> { + let (root, paths) = test_paths("restart-lock-until-published"); + paths.ensure()?; + let service_id = "svc-restart-lock-until-published"; + let record = stop_test_record(&paths, service_id); + record.write()?; + + READINESS_TIMEOUT_OVERRIDE.with(|cell| cell.set(Some(Duration::ZERO))); + let (observed_tx, observed_rx) = std::sync::mpsc::channel(); + let lock_path = paths.managed_launch_lock_path(); + let manifest_path = paths.service_manifest_path(service_id); + let spawn: &RestartSpawnOverride<'_> = &|command| { + let child = std::process::Command::new("sleep") + .arg("30") + .spawn() + .map_err(|error| io::Error::other(format!("{error} ({command:?})")))?; + let (lock_path, manifest_path, tx) = ( + lock_path.clone(), + manifest_path.clone(), + observed_tx.clone(), + ); + thread::spawn(move || { + // Blocks until the restart drops the lock. + let _lock = rocm_core::FileLock::acquire(&lock_path).expect("acquire"); + let record: ManagedServiceRecord = + serde_json::from_slice(&fs::read(manifest_path).expect("read")).expect("parse"); + let _ = tx.send(record.supervisor_pid); + }); + Ok(child) + }; + let result = restart_internal_managed_service_inner( + &paths, + service_id, + None, + |_| (Vec::new(), true), + Some(spawn), + ); + READINESS_TIMEOUT_OVERRIDE.with(|cell| cell.set(None)); + let started = result?; + let observed = observed_rx.recv_timeout(Duration::from_secs(10)); + // Reap the stand-in engine. + let _ = std::process::Command::new("kill") + .arg(started.supervisor_pid.to_string()) + .status(); + let _ = fs::remove_dir_all(root); + + assert_eq!( + observed.expect("the waiter must get the lock after the restart"), + started.supervisor_pid, + "the lock was released before the record named the restarted child" + ); + Ok(()) + } + + #[cfg(not(windows))] + #[test] + fn restart_is_not_blocked_by_a_live_record_on_another_port() -> Result<()> { + // `blocks_port` compares ports. A live service elsewhere must not trip + // the "claimed that port" refusal: if it ignored the port, every + // restart would be refused while any other service ran. + let (root, paths) = test_paths("restart-live-other-port"); + paths.ensure()?; + let service_id = "svc-restart-live-other-port"; + let record = stop_test_record(&paths, service_id); + let host = record.host.clone(); + let port = record.port; + record.write()?; + + let mut other = ManagedServiceRecord::new( + &paths, + "svc-live-other-port", + "lemonade", + "other-model", + "other-model-canonical", + &host, + port.wrapping_add(1), + "managed", + 0, + None, + None, + None, + ); + other.status = "starting".to_owned(); + other.write()?; + + let spawn: &RestartSpawnOverride<'_> = + &|_command| Err(io::Error::other("spawn intercepted by test")); + let result = restart_internal_managed_service_inner( + &paths, + service_id, + None, + |_| (Vec::new(), true), + Some(spawn), + ); + let _ = fs::remove_dir_all(root); + let rendered = format!("{:#}", result.expect_err("the intercepted spawn must fail")); + assert!( + rendered.contains("spawn intercepted by test"), + "a live service on another port must not block the restart: {rendered}" + ); + Ok(()) + } + /// The value following `flag` in an argv, or `None` when the flag is absent. fn argv_flag_value<'a>(args: &'a [String], flag: &str) -> Option<&'a str> { let index = args.iter().position(|arg| arg == flag)?; diff --git a/apps/rocmd/src/common.rs b/apps/rocmd/src/common.rs index 5705d27c5..6a83806da 100644 --- a/apps/rocmd/src/common.rs +++ b/apps/rocmd/src/common.rs @@ -326,7 +326,7 @@ pub(crate) fn ensure_public_service_has_endpoint_key( bail!( "refusing to respawn a service bound to the public host `{host}` without an endpoint \ API key: it would come back up without authentication. Relaunch it with \ - `rocm serve --host {host} --allow-public-bind` to issue a new key." + `rocm serve --host {host} --port --allow-public-bind` to issue a new key." ); } Ok(()) diff --git a/apps/rocmd/src/mcp.rs b/apps/rocmd/src/mcp.rs index bffe3fae4..6412242a9 100644 --- a/apps/rocmd/src/mcp.rs +++ b/apps/rocmd/src/mcp.rs @@ -445,7 +445,8 @@ pub(crate) fn rocm_mcp_tools() -> Vec { "port": { "type": "integer", "minimum": 1, - "maximum": 65535 + "maximum": 65535, + "description": "Required unless host is exactly 127.0.0.1 (localhost/::1 still need one)." }, "allow_public_bind": { "type": "boolean" @@ -1079,6 +1080,20 @@ fn build_launch_server_args(arguments: &serde_json::Map) -> Resul "launch_server host `{host}` is not loopback; require `allow_public_bind=true` before binding a non-local interface" ); } + // A malformed `port` must be rejected here, not silently dropped by the + // argv builder and left to fall through to automatic port selection. + let port = rocm_core::launch_port_argument(arguments.get("port")) + .map_err(|reason| anyhow::anyhow!("launch_server argument `port` {reason}"))?; + // `rocm serve` proves a port is free only on the exact default host; every + // other spelling (including the loopback aliases `localhost`/`::1` the + // check above still accepts) needs an explicit port or the CLI call + // refuses later with a less actionable error. See `is_canonical_loopback_host`. + if port.is_none() && !rocm_core::is_canonical_loopback_host(host) { + bail!( + "launch_server host `{host}` needs an explicit `port`; automatic port selection is \ + only supported on the default host {DEFAULT_LOCAL_HOST}" + ); + } let mut argv = vec!["serve".to_owned(), model.to_owned(), "--managed".to_owned()]; if let Some(engine) = arguments.get("engine").and_then(Value::as_str) { @@ -1102,7 +1117,7 @@ fn build_launch_server_args(arguments: &serde_json::Map) -> Resul if allow_public_bind { argv.push("--allow-public-bind".to_owned()); } - if let Some(port) = arguments.get("port").and_then(Value::as_u64) { + if let Some(port) = port { argv.push("--port".to_owned()); argv.push(port.to_string()); } @@ -1407,12 +1422,65 @@ mod tests { ("model".to_owned(), Value::String("tiny-gpt2".to_owned())), ("host".to_owned(), Value::String("0.0.0.0".to_owned())), ("allow_public_bind".to_owned(), Value::Bool(true)), + ("port".to_owned(), Value::Number(11999.into())), ]); let args = build_launch_server_args(&arguments)?; assert!(args.contains(&"--allow-public-bind".to_owned())); Ok(()) } + #[test] + fn launch_server_requires_an_explicit_port_for_non_canonical_loopback_hosts() { + let arguments = serde_json::Map::from_iter([ + ("model".to_owned(), Value::String("tiny-gpt2".to_owned())), + ("host".to_owned(), Value::String("localhost".to_owned())), + ]); + let error = build_launch_server_args(&arguments).unwrap_err(); + assert!(error.to_string().contains("explicit `port`"), "{error:#}"); + } + + #[test] + fn launch_server_rejects_a_malformed_port_instead_of_choosing_one_automatically() { + // Each value would be dropped by a lenient argv builder and leave the + // launch to automatic port selection on a host that cannot do it. + for (port, expected) in [ + (Value::String("8000".to_owned()), "must be an integer"), + (Value::Number(0.into()), "between 1 and 65535"), + (Value::Number(65536.into()), "between 1 and 65535"), + ] { + let arguments = serde_json::Map::from_iter([ + ("model".to_owned(), Value::String("tiny-gpt2".to_owned())), + ("host".to_owned(), Value::String("localhost".to_owned())), + ("port".to_owned(), port.clone()), + ]); + let error = build_launch_server_args(&arguments).unwrap_err(); + assert!(error.to_string().contains(expected), "{port}: {error:#}"); + } + } + + #[test] + fn launch_server_treats_a_null_port_as_automatic() -> Result<()> { + let arguments = serde_json::Map::from_iter([ + ("model".to_owned(), Value::String("tiny-gpt2".to_owned())), + ("port".to_owned(), Value::Null), + ]); + let args = build_launch_server_args(&arguments)?; + assert!(!args.contains(&"--port".to_owned()), "{args:?}"); + Ok(()) + } + + #[test] + fn launch_server_allows_localhost_with_an_explicit_port() -> Result<()> { + let arguments = serde_json::Map::from_iter([ + ("model".to_owned(), Value::String("tiny-gpt2".to_owned())), + ("host".to_owned(), Value::String("localhost".to_owned())), + ("port".to_owned(), Value::Number(11999.into())), + ]); + let args = build_launch_server_args(&arguments)?; + assert!(args.contains(&"11999".to_owned())); + Ok(()) + } + #[test] fn install_sdk_rejects_system_prefix_without_ack() { let arguments = serde_json::Map::from_iter([( diff --git a/apps/rocmd/src/test_support.rs b/apps/rocmd/src/test_support.rs index f1c000bfe..a20aea292 100644 --- a/apps/rocmd/src/test_support.rs +++ b/apps/rocmd/src/test_support.rs @@ -94,6 +94,18 @@ pub(crate) fn assert_each_call_gets_its_own_root(make_root: impl Fn() -> PathBuf ); } +/// A loopback port the OS just handed out and nothing holds now. +/// +/// For tests that make the code under test bind the port for real: a fixed +/// number inside the automatic range 11435-11535 fails whenever a real +/// `rocm serve` is running on the developer's machine. +pub(crate) fn free_loopback_port() -> u16 { + std::net::TcpListener::bind(("127.0.0.1", 0)) + .and_then(|listener| listener.local_addr()) + .expect("bind an ephemeral loopback port") + .port() +} + /// Build the record a supervisor would construct for `service_id`, to learn /// the paths it derives before the real call does. Not platform-gated: /// the supervisor exit-write tests use it on every platform. diff --git a/apps/rocmd/src/watchers.rs b/apps/rocmd/src/watchers.rs index 5ec222daa..3eedb5bda 100644 --- a/apps/rocmd/src/watchers.rs +++ b/apps/rocmd/src/watchers.rs @@ -1333,7 +1333,34 @@ fn handle_server_recover_event_with_record( Some(record.service_id.clone()), ); } - restart_managed_service(paths, &mut *record)?; + // A contended launch lock or an occupied recorded port is a + // recoverable, per-service condition — report it and move on rather + // than letting one service take the whole 30s watcher tick (and the + // daemon) down. + if let Err(error) = restart_managed_service(paths, &mut *record) { + // `restart_managed_service` can fail before it ever writes the + // record (e.g. the recorded port is still occupied), so without + // this stamp the backoff guard above never sees a prior attempt + // and the next 30s tick retries immediately, forever. Best + // effort: a write failure here just means the next tick retries + // sooner, not a worse outcome than today. + let _ = crate::service::update_supervised_record(paths, record, |current| { + current.last_restart_unix_ms = Some(now); + }); + return crate::persistence::record_event( + paths, + state, + "server-recover", + "error", + "restart_managed_service_failed", + &format!( + "failed to recover managed service {} on {}:{} after \ + {recovery_reason_display}: {error:#}", + record.service_id, record.host, record.port + ), + Some(record.service_id.clone()), + ); + } crate::persistence::record_event( paths, state, @@ -1412,6 +1439,15 @@ pub(crate) fn find_recoverable_service( if record.mode != "managed" || stop_requested(&record) { continue; } + // A candidate still inside its restart backoff would only be skipped by + // the caller; returning it here would starve every service after it + // while a refused recovery (e.g. its port was taken) keeps re-stamping. + if record + .last_restart_unix_ms + .is_some_and(|last| now.saturating_sub(last) < SERVER_RECOVER_BACKOFF_MS) + { + continue; + } if let Some(reason) = manifest_service_recovery_reason(&record, now) { return Ok(Some((record, reason))); } @@ -1501,10 +1537,136 @@ fn manifest_service_recovery_reason( } } +/// Respawn the supervisor for a managed service that needs recovery. +/// +/// Recovery re-claims the concrete port already written in the record under +/// [`rocm_core::MANAGED_LAUNCH_LOCK_TIMEOUT`]-bounded acquisition of the same +/// cross-process launch lock a `rocm serve` launch holds (unbounded, via +/// [`rocm_core::FileLock::acquire`]). The two still can't publish competing +/// records for this port at the same instant — the lock is exclusive either +/// way. A concurrent `rocm serve` launch does still block for as long as +/// recovery holds the lock; what the bound buys is the other direction — +/// recovery never waits out an interactive launch indefinitely. If a `rocm +/// serve` call is already mid-transaction when this bounded acquire times +/// out, recovery is deferred (not retried in a busy loop) rather than +/// queuing behind the CLI. pub(crate) fn restart_managed_service( paths: &AppPaths, record: &mut ManagedServiceRecord, ) -> Result<()> { + restart_managed_service_with(paths, record, None) +} + +/// Replaces the real `command.spawn()` of a recovery supervisor: a test-only +/// seam for asserting on state at the exact instant production would spawn. +type SpawnOverride<'a> = dyn Fn(&mut ProcessCommand) -> std::io::Result + 'a; + +/// [`restart_managed_service`], with the supervisor spawn interceptable via +/// `spawn_override` (the real `command.spawn()` when `None`). +fn restart_managed_service_with( + paths: &AppPaths, + record: &mut ManagedServiceRecord, + spawn_override: Option<&SpawnOverride<'_>>, +) -> Result<()> { + restart_managed_service_bounded( + paths, + record, + spawn_override, + rocm_core::MANAGED_LAUNCH_LOCK_TIMEOUT, + ) +} + +/// [`restart_managed_service_with`] with the launch-lock wait injectable, so a +/// test can prove the wait is bounded without waiting out the real bound. +fn restart_managed_service_bounded( + paths: &AppPaths, + record: &mut ManagedServiceRecord, + spawn_override: Option<&SpawnOverride<'_>>, + lock_timeout: Duration, +) -> Result<()> { + let _launch_lock = + rocm_core::FileLock::acquire_bounded(paths.managed_launch_lock_path(), lock_timeout)?; + + // Re-check against the disk, now that the lock excludes a concurrent + // `rocm serve` publish: the caller's `record` was read before this lock was + // acquired, so a fresh launch could have claimed this exact (host, port) + // in the meantime (the engine itself binds the port only after the model + // loads, so a raw OS bind probe alone would not catch that — see + // `crate::persistence::load_managed_services`'s callers for the shared + // `is_live` reservation this mirrors). Refuse rather than spawn a + // competitor; the next watcher tick reconsiders this service. + let on_disk = crate::persistence::load_managed_services(paths) + .context("failed to re-read managed services under the launch lock")?; + if let Some(collision) = on_disk + .iter() + .find(|other| other.service_id != record.service_id && other.blocks_port(record.port)) + { + anyhow::bail!( + "cannot recover managed service {} on {}:{}: {} claimed that port while recovery \ + was waiting for the launch lock", + record.service_id, + record.host, + record.port, + collision.service_id + ); + } + + // The same re-read must also prove this service's own record is still the + // one the caller saw: a `rocm services restart` of this service can finish + // while recovery waits for the lock, and writing the stale copy over it + // would orphan the restarted engine or start a second supervisor. + match load_service_record(paths, &record.service_id) { + Ok(current) => { + // Only a stop newer than the caller's copy counts: restarting a + // record that already carried one deliberately supersedes it. + if stop_requested(¤t) + && current.stop_requested_unix_ms != record.stop_requested_unix_ms + { + anyhow::bail!( + "cannot recover managed service {}: a stop was requested while recovery was \ + waiting for the launch lock", + record.service_id + ); + } + if current.status != record.status + || current.supervisor_pid != record.supervisor_pid + || current.supervisor_start_ticks != record.supervisor_start_ticks + || current.engine_pid != record.engine_pid + { + anyhow::bail!( + "cannot recover managed service {}: its record changed while recovery was \ + waiting for the launch lock", + record.service_id + ); + } + } + Err(_) if !paths.service_manifest_path(&record.service_id).exists() => { + anyhow::bail!( + "cannot recover managed service {}: it was removed while recovery was waiting \ + for the launch lock", + record.service_id + ); + } + Err(error) => return Err(error.context("failed to re-read the service record")), + } + + // On the canonical loopback host, prove the recorded port is genuinely free + // before publishing `recovering` or spawning anything: a supervisor that + // cannot bind would replace a truthful record with a lie. Custom hosts keep + // the existing engine-owned bind behavior — the record transition is still + // serialized, but the bind is not preflighted. + let lease = if rocm_core::is_canonical_loopback_host(&record.host) { + Some(rocm_core::lease_loopback_port_or_bind_error( + record.port, + format_args!( + "cannot recover managed service {} on {}:{}: its recorded port is in use", + record.service_id, record.host, record.port + ), + )?) + } else { + None + }; + let rocmd_binary = std::env::current_exe().context("failed to resolve current rocmd executable path")?; let log_file = fs::OpenOptions::new() @@ -1516,6 +1678,13 @@ pub(crate) fn restart_managed_service( .try_clone() .context("failed to clone service log file handle")?; + // The truthful state to fall back to. Every failure after the `recovering` + // write below (spawn, `try_wait`, recording the supervisor) restores this + // through `fail_recovery` rather than leave a half-applied `recovering` row + // that would keep reserving the port, and a supervisor that exists is never + // left running unrecorded. + let prior = record.clone(); + record.status = "recovering".to_owned(); // Counts the restart and drops the previous run's inference verification. // The respawned child writes a fresh record of its own, and "recovering" is @@ -1543,23 +1712,79 @@ pub(crate) fn restart_managed_service( // Restarting is a request to run, so it supersedes any earlier stop request, // as `rocm services restart` treats it too. record.stop_requested_unix_ms = None; - record.write()?; + if let Err(error) = record.write() { + // Nothing was spawned and the on-disk record was never replaced; just + // undo the in-memory transition so the caller does not report recovery + // that never happened. + *record = prior; + return Err(error.context("failed to publish the recovering service record")); + } + + // Hand the port to the supervisor: close the probe socket immediately before + // the spawn, still holding the launch lock. + if let Some(lease) = lease { + lease.release(); + } - let mut child = detached_rocmd_command(&rocmd_binary) + let recovering_snapshot = record.clone(); + let mut command = detached_rocmd_command(&rocmd_binary); + command .args(recovery_supervise_args(record)) .stdin(Stdio::null()) .stdout(Stdio::from(log_file)) - .stderr(Stdio::from(log_file_err)) - .spawn() - .context("failed to spawn recovery supervisor")?; + .stderr(Stdio::from(log_file_err)); + let spawned = match spawn_override { + Some(spawn) => spawn(&mut command), + None => command.spawn(), + }; + let mut child = match spawned { + Ok(child) => child, + Err(error) => { + let error = anyhow::Error::new(error).context("failed to spawn recovery supervisor"); + return Err(fail_recovery( + paths, + record, + &recovering_snapshot, + prior, + error, + )); + } + }; - record_spawned_supervisor(paths, record, child.id())?; + if let Err(error) = record_spawned_supervisor(paths, record, child.id()) { + // The supervisor is alive but unrecorded: it would hold the port while + // being invisible to every stop/recovery path. Kill it first. + terminate_recovery_supervisor(&mut child); + let error = error.context("failed to publish the recovering service record"); + return Err(fail_recovery( + paths, + record, + &recovering_snapshot, + prior, + error, + )); + } + // The supervisor's identity is on disk now, so the rollback guard must + // compare against this copy, not the one from before it was recorded. + let recorded_snapshot = record.clone(); thread::sleep(Duration::from_millis(200)); - if let Some(status) = child - .try_wait() - .context("failed to check recovery supervisor startup state")? - { + let exited = match child.try_wait() { + Ok(exited) => exited, + Err(error) => { + terminate_recovery_supervisor(&mut child); + let error = anyhow::Error::new(error) + .context("failed to check recovery supervisor startup state"); + return Err(fail_recovery( + paths, + record, + &recorded_snapshot, + prior, + error, + )); + } + }; + if let Some(status) = exited { // Not a write-back of `record`: a stop issued since the spawn has // persisted its marker, and a `failed` record without one is what // recovery restarts. Same discipline as the supervisor's own writes. @@ -1575,8 +1800,69 @@ pub(crate) fn restart_managed_service( Ok(()) } -/// Record the supervisor [`restart_managed_service`] just spawned, as `pid`. +/// Undo [`restart_managed_service`]'s `recovering` publication after a later +/// step failed, and return the error to report. +/// +/// Not a blind `record.write()` of `prior`: the same discipline as +/// `update_supervised_record` applies, so a stop marker a concurrent stop +/// persisted since the first write is not reverted. When the rollback itself +/// fails the on-disk record still claims `recovering` for a supervisor that no +/// longer exists, which is worse than the original failure and is reported with +/// it rather than swallowed. +fn fail_recovery( + paths: &AppPaths, + record: &mut ManagedServiceRecord, + recovering_snapshot: &ManagedServiceRecord, + prior: ManagedServiceRecord, + error: anyhow::Error, +) -> anyhow::Error { + if let Err(rollback) = rollback_recovering_record(paths, recovering_snapshot, &prior) { + return error.context(format!( + "rollback also failed: managed service {} is left recorded as recovering \ + with no supervisor ({rollback:#})", + record.service_id + )); + } + *record = prior; + error +} + +/// Grace given to a recovery supervisor that must be terminated because its +/// record could not be published. Short: it is milliseconds old. +const RECOVERY_CLEANUP_GRACE: Duration = Duration::from_secs(2); + +// Test-only, thread-scoped call counter: `restart_managed_service`'s rollback +// branch asserts this to prove the kill is actually reached (not just that the +// record comes back truthful), since that branch's own supervisor is a +// re-exec'd test binary that exits on its own almost immediately regardless of +// whether it is killed, so `process_is_running` on it would not catch the call +// being dropped. Thread-local, not a global counter: cargo runs tests +// concurrently on separate threads, and this function has another, unrelated +// direct caller (`terminate_recovery_supervisor_kills_and_reaps_the_child`). +#[cfg(test)] +thread_local! { + static TERMINATE_RECOVERY_SUPERVISOR_CALLS: std::cell::Cell = + const { std::cell::Cell::new(0) }; +} + +/// Terminate and reap a recovery supervisor that could not be recorded. /// +/// Tree scope, forced: the supervisor may already have started an engine child, +/// and leaving either alive would keep the recorded port occupied by a process +/// no stop or recovery path can find. Reaping afterwards keeps the daemon free +/// of zombies. +fn terminate_recovery_supervisor(child: &mut std::process::Child) { + #[cfg(test)] + TERMINATE_RECOVERY_SUPERVISOR_CALLS.with(|calls| calls.set(calls.get() + 1)); + let _outcome = rocm_core::terminate_verified( + &rocm_core::ProcessIdentity::capture(child.id()), + rocm_core::KillScope::Tree, + RECOVERY_CLEANUP_GRACE, + true, + ); + let _ = child.wait(); +} + /// Between the restart's first write — which clears the recorded PIDs, since /// no supervisor exists yet — and this one, the record names no process at /// all. A stop that starts in that window persists its marker first and then @@ -1603,6 +1889,49 @@ fn record_spawned_supervisor( }) } +/// Restore only the fields [`restart_managed_service`]'s transaction owns, +/// after `record_spawned_supervisor` failed to publish them. The same +/// discipline as `update_supervised_record` elsewhere in that function: a +/// stop marker a concurrent stop persisted on the live record must survive +/// this rollback, not be overwritten by a blind copy of `restore`. +/// +/// Differs from that helper in one case: when the manifest cannot be parsed +/// at all (as opposed to merely naming a different supervisor), there is no +/// live data left to selectively preserve, so this falls back to writing +/// `restore` in full rather than no-op'ing. +/// `restart_rolls_back_and_kills_the_supervisor_when_the_second_write_cannot_be_read` +/// exercises exactly that fallback. +fn rollback_recovering_record( + paths: &AppPaths, + recovering_snapshot: &ManagedServiceRecord, + restore: &ManagedServiceRecord, +) -> Result<()> { + let manifest_path = paths.service_manifest_path(&recovering_snapshot.service_id); + if !manifest_path.exists() { + return Ok(()); + } + match load_service_record(paths, &recovering_snapshot.service_id) { + Ok(mut current) + if current.supervisor_pid == recovering_snapshot.supervisor_pid + && current.supervisor_start_ticks == recovering_snapshot.supervisor_start_ticks => + { + current.status = restore.status.clone(); + current.supervisor_pid = restore.supervisor_pid; + current.supervisor_start_ticks = restore.supervisor_start_ticks; + current.engine_pid = restore.engine_pid; + current.engine_start_ticks = restore.engine_start_ticks; + current.restart_count = restore.restart_count; + current.last_restart_unix_ms = restore.last_restart_unix_ms; + current.inference_verified_at_unix_ms = restore.inference_verified_at_unix_ms; + current.inference_probe_attempted_at_unix_ms = + restore.inference_probe_attempted_at_unix_ms; + current.write() + } + Ok(_) => Ok(()), + Err(_) => restore.write(), + } +} + fn recovery_supervise_args(record: &ManagedServiceRecord) -> Vec { let mut args = vec![ "supervise".to_owned(), @@ -1728,10 +2057,10 @@ fn detached_rocmd_command(rocmd_binary: &std::path::Path) -> ProcessCommand { #[cfg(test)] mod tests { use super::*; - use crate::test_support::identity_probe_record; use crate::test_support::{ UNCONFIRMED_STOP_PID, seed_keyed_service, stop_with_outcome, temp_app_paths, }; + use crate::test_support::{free_loopback_port, identity_probe_record}; use rocm_core::{AuditEventRecord, AutomationEventRecord}; fn test_watcher_snapshot( @@ -3540,6 +3869,318 @@ mod tests { ); } + #[test] + fn recovery_refuses_a_port_a_launch_claimed_while_it_waited_for_the_lock() -> Result<()> { + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + + // The gap this closes: a crashed service's port is simultaneously + // "free" to a brand-new `rocm serve --gpu auto` launch (which only + // checks *live* records) and "needs recovery" to this function (which + // used to only probe the raw OS bind). Neither process's engine binds + // the port until its model finishes loading, so a probe alone cannot + // see a `starting` competitor that has not bound yet. Recreate that + // exact disk state under the lock and require recovery to refuse, + // rather than racing a second engine onto the same port. + let (_root, paths) = temp_app_paths("recovery-launch-claimed-port"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let port = 21_777_u16; + let mut record = ManagedServiceRecord::new( + &paths, + "lemonade-recover-3", + "lemonade", + "qwen", + "qwen-canonical".to_owned(), + rocm_core::DEFAULT_LOCAL_HOST, + port, + "managed", + 0, + None, + None, + None, + ); + record.status = "failed".to_owned(); + record.write()?; + + let guard = rocm_core::FileLock::acquire_bounded( + paths.managed_launch_lock_path(), + rocm_core::MANAGED_LAUNCH_LOCK_TIMEOUT, + )?; + let finished = Arc::new(AtomicBool::new(false)); + let handle = { + let paths = paths.clone(); + let finished = Arc::clone(&finished); + thread::spawn(move || { + let result = restart_managed_service(&paths, &mut record); + finished.store(true, Ordering::SeqCst); + result + }) + }; + + thread::sleep(Duration::from_millis(250)); + assert!( + !finished.load(Ordering::SeqCst), + "recovery must serialize on the shared launch lock" + ); + + // Simulate a `rocm serve --gpu auto` publishing a brand-new service on + // this exact port while recovery was still waiting for the lock it + // itself now holds — the one window the launch lock is supposed to + // close. + let mut competitor = ManagedServiceRecord::new( + &paths, + "lemonade-fresh-launch", + "lemonade", + "other-model", + "other-model-canonical".to_owned(), + rocm_core::DEFAULT_LOCAL_HOST, + port, + "managed", + 0, + None, + None, + None, + ); + competitor.status = "starting".to_owned(); + competitor.write()?; + drop(guard); + + let error = handle + .join() + .expect("recovery thread") + .expect_err("a live competitor on the same port must refuse recovery"); + let message = format!("{error:#}"); + assert!(message.contains("lemonade-fresh-launch"), "{message}"); + assert!(message.contains("claimed that port"), "{message}"); + + let on_disk = crate::persistence::load_managed_services(&paths)? + .into_iter() + .find(|candidate| candidate.service_id == "lemonade-recover-3") + .expect("record still present"); + assert_eq!( + on_disk.status, "failed", + "refused recovery must not overwrite the truthful record with `recovering`" + ); + Ok(()) + } + + #[cfg(target_os = "linux")] + #[test] + fn recovery_is_not_blocked_by_a_stopped_or_failed_record_on_the_same_port() -> Result<()> { + // The mirror image of the refusal above: `blocks_port` is + // `is_live() && port == port`, so a `stopped` or `failed` record on + // the same port must not count as a collision. Mutation-proof: if + // `blocks_port` regressed to ignore status, either sibling below + // would trip the "claimed that port" bail and this test would go red. + let (root, paths) = temp_app_paths("recovery-non-live-same-port"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let port = free_loopback_port(); + let mut stopped_sibling = ManagedServiceRecord::new( + &paths, + "lemonade-stopped-sibling", + "lemonade", + "other-model", + "other-model-canonical".to_owned(), + rocm_core::DEFAULT_LOCAL_HOST, + port, + "managed", + 0, + None, + None, + None, + ); + stopped_sibling.status = "stopped".to_owned(); + stopped_sibling.write()?; + + let mut failed_sibling = ManagedServiceRecord::new( + &paths, + "lemonade-failed-sibling", + "lemonade", + "other-model-2", + "other-model-2-canonical".to_owned(), + rocm_core::DEFAULT_LOCAL_HOST, + port, + "managed", + 0, + None, + None, + None, + ); + failed_sibling.status = "failed".to_owned(); + failed_sibling.write()?; + + let mut record = identity_probe_record(&paths, "lemonade-recover-non-live", port); + record.status = "failed".to_owned(); + record.write()?; + + // The re-exec'd "supervisor" is this very test binary, which cannot + // actually run as `rocmd supervise` and exits almost immediately; the + // same limitation `restart_managed_service_persists_the_supervisor_identity_token` + // works around. What this proves is that the spawn was ever reached: + // a wrongly-blocking `blocks_port` would bail before `record.write()` + // below ever runs, leaving `supervisor_start_ticks` at `None`. + let result = restart_managed_service(&paths, &mut record); + let persisted = load_service_record(&paths, "lemonade-recover-non-live"); + fs::remove_dir_all(root).ok(); + + if let Err(error) = &result { + let message = format!("{error:#}"); + assert!( + !message.contains("claimed that port"), + "a stopped or failed sibling on the same port must not be treated as a collision: {message}" + ); + } + assert!( + persisted?.supervisor_start_ticks.is_some(), + "recovery must reach the spawn, proving neither sibling blocked it" + ); + Ok(()) + } + + #[test] + fn contained_recovery_survives_a_failed_restart_and_backs_off() -> Result<()> { + use std::net::TcpListener; + + let (root, paths) = temp_app_paths("recovery-contained-port-collision"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let occupant = TcpListener::bind(("127.0.0.1", 0))?; + let port = occupant.local_addr()?.port(); + + let mut record = ManagedServiceRecord::new( + &paths, + "lemonade-recover-2", + "lemonade", + "qwen", + "qwen-canonical".to_owned(), + rocm_core::DEFAULT_LOCAL_HOST, + port, + "managed", + 0, + None, + None, + None, + ); + record.status = "failed".to_owned(); + record.write()?; + + let mut state = test_runtime_state(vec![test_watcher_snapshot( + "server-recover", + WatcherMode::Contained, + None, + )]); + let event = AutomationTriggerEvent { + at_unix_ms: 1, + kind: "service.manifest_recoverable".to_owned(), + source: "manifest_scan".to_owned(), + watcher_hint: Some("server-recover".to_owned()), + service_id: Some(record.service_id.clone()), + reason: Some("manifest_status_failed".to_owned()), + payload: json!({}), + }; + + // The watcher must not propagate a per-service failure: that would take + // the whole 30s tick (and every other watcher) down with it. + handle_server_recover_event_with_record( + &paths, + WatcherMode::Contained, + &mut state, + &event, + &mut record, + ) + .expect("a failed restart must be caught and reported, not propagated"); + + let event_text = fs::read_to_string(paths.automation_events_path())?; + let logged = serde_json::from_str::(event_text.trim())?; + let reloaded = load_service_record(&paths, "lemonade-recover-2")?; + fs::remove_dir_all(root).ok(); + drop(occupant); + + assert_eq!(logged.level, "error"); + assert_eq!(logged.action, "restart_managed_service_failed"); + assert!( + logged.message.contains("recorded port is in use"), + "{}", + logged.message + ); + // Unchanged other than the backoff stamp: the occupied port must not + // have been reported as recovered. + assert_eq!(reloaded.status, "failed"); + // Stamped despite `restart_managed_service` never writing the record on + // this failure path, so the next 30s tick's backoff guard sees a prior + // attempt instead of retrying immediately forever. + assert!(reloaded.last_restart_unix_ms.is_some()); + Ok(()) + } + + #[test] + fn recovery_refuses_when_the_registry_cannot_be_re_read_leaving_the_prior_record_intact() + -> Result<()> { + use std::net::TcpListener; + + let (_root, paths) = temp_app_paths("recovery-publication-failure"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + // A free recorded port, so a lease was never the obstacle under test. + let probe = TcpListener::bind(("127.0.0.1", 0))?; + let port = probe.local_addr()?.port(); + drop(probe); + + let mut record = ManagedServiceRecord::new( + &paths, + "lemonade-recover-2", + "lemonade", + "qwen", + "qwen-canonical".to_owned(), + rocm_core::DEFAULT_LOCAL_HOST, + port, + "managed", + 4242, + None, + None, + None, + ); + record.status = "failed".to_owned(); + record.write()?; + + // Replace the manifest with a directory. `load_managed_services` scans + // every `.json` entry under `services_dir()` to re-check for a + // collision before publishing `recovering`, and this record's own + // manifest is one of those entries — so the corruption trips that + // re-read, not the publish a few lines later. That is the more + // conservative of the two and still proves the invariant this test is + // for: recovery must fail closed, not spawn a competitor, when it + // cannot trust what else is live. + fs::remove_file(&record.manifest_path)?; + fs::create_dir_all(&record.manifest_path)?; + + let mut attempt = record.clone(); + let error = restart_managed_service(&paths, &mut attempt) + .expect_err("an unreadable registry must refuse, not assume no collision"); + let message = format!("{error:#}"); + assert!( + message.contains("failed to re-read managed services under the launch lock"), + "{message}" + ); + // Nothing was spawned and the on-disk row was never replaced, so the + // error must NOT claim a failed rollback: that phrasing is reserved for + // the post-spawn case where the record really is left mid-transition. + assert!(!message.contains("rollback also failed"), "{message}"); + + // The caller's record must not claim a recovery that never happened. + assert_eq!(attempt.status, "failed"); + assert_eq!(attempt.restart_count, record.restart_count); + assert_eq!(attempt.supervisor_pid, record.supervisor_pid); + fs::remove_dir_all(&record.manifest_path)?; + Ok(()) + } + /// The same guarantee for the daemon's recovery path, which records a PID of /// its own. See `supervise_service_persists_the_supervisor_identity_token` /// for why an unrecorded token is the defect rather than a cosmetic gap. @@ -3555,7 +4196,7 @@ mod tests { fs::create_dir_all(paths.services_dir())?; let service_id = "svc-restart-identity"; - let mut record = identity_probe_record(&paths, service_id, 11444); + let mut record = identity_probe_record(&paths, service_id, free_loopback_port()); record.status = "failed".to_owned(); record.write()?; assert_eq!( @@ -3575,6 +4216,54 @@ mod tests { Ok(()) } + /// Mirrors `launch_releases_the_leased_port_before_spawning_the_child` in + /// `apps/rocm/src/service_launch.rs`: the probe lease on the recorded port + /// must already be released by the time the recovery supervisor is + /// spawned, or the supervisor's own bind of that port would fail. + #[test] + fn restart_releases_the_leased_port_before_spawning_the_supervisor() -> Result<()> { + use std::cell::Cell; + use std::net::TcpListener; + + let (root, paths) = temp_app_paths("restart-releases-lease"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let probe = TcpListener::bind(("127.0.0.1", 0))?; + let port = probe.local_addr()?.port(); + drop(probe); + + let service_id = "svc-restart-releases-lease"; + let mut record = identity_probe_record(&paths, service_id, port); + record.status = "failed".to_owned(); + record.write()?; + + let bind_succeeded = Cell::new(false); + let spawn_override = + |command: &mut ProcessCommand| -> std::io::Result { + // Retry briefly: a sibling test's concurrent fork can hold a + // copy of the just-released socket until its exec closes it. + let deadline = std::time::Instant::now() + Duration::from_secs(2); + while !bind_succeeded.get() && std::time::Instant::now() < deadline { + bind_succeeded.set(TcpListener::bind(("127.0.0.1", port)).is_ok()); + } + command.spawn() + }; + + // The re-exec'd "supervise" args are rejected by this test binary, + // which exits at once (see + // restart_managed_service_persists_the_supervisor_identity_token), so + // the restart itself is expected to report failure here. + let _ = restart_managed_service_with(&paths, &mut record, Some(&spawn_override)); + fs::remove_dir_all(root).ok(); + + assert!( + bind_succeeded.get(), + "the leased port must already be bindable (released) by the time the supervisor is spawned" + ); + Ok(()) + } + /// An unconfirmed stop leaves the status at `ready` on purpose, which is /// exactly what the daemon's recovery scan looks for. The stop request it /// records is the only thing telling it apart from a crash, so recovery @@ -3687,7 +4376,7 @@ mod tests { fs::create_dir_all(paths.services_dir())?; let service_id = "svc-restart-drops-engine-pid"; - let mut record = identity_probe_record(&paths, service_id, 11455); + let mut record = identity_probe_record(&paths, service_id, free_loopback_port()); // The shape `rocm serve --background` writes: one PID in both roles. record.supervisor_pid = UNCONFIRMED_STOP_PID; record.engine_pid = Some(UNCONFIRMED_STOP_PID); @@ -3714,6 +4403,694 @@ mod tests { Ok(()) } + /// Recovery re-claims the concrete port already written in the record, so + /// it must serialize on the same cross-process launch lock a fresh + /// `rocm serve` launch takes, and must refuse a recorded port something + /// else already holds rather than publish a `recovering` record that + /// lies. + #[test] + fn recovery_waits_for_the_allocation_lock_and_refuses_an_occupied_port() -> Result<()> { + use std::net::TcpListener; + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + + let (root, paths) = temp_app_paths("recovery-port-collision"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + // Something else already owns the port this service recorded. + let occupant = TcpListener::bind(("127.0.0.1", 0))?; + let port = occupant.local_addr()?.port(); + + let mut record = identity_probe_record(&paths, "svc-recovery-port-collision", port); + record.status = "failed".to_owned(); + record.write()?; + + let guard = rocm_core::FileLock::acquire_bounded( + paths.managed_launch_lock_path(), + rocm_core::MANAGED_LAUNCH_LOCK_TIMEOUT, + )?; + let finished = Arc::new(AtomicBool::new(false)); + let handle = { + let paths = paths.clone(); + let finished = Arc::clone(&finished); + let mut record = record.clone(); + thread::spawn(move || { + let result = restart_managed_service(&paths, &mut record); + finished.store(true, Ordering::SeqCst); + result + }) + }; + + thread::sleep(Duration::from_millis(250)); + assert!( + !finished.load(Ordering::SeqCst), + "recovery must serialize on the shared launch lock" + ); + drop(guard); + + let error = handle + .join() + .expect("recovery thread") + .expect_err("an occupied recorded port must fail recovery"); + let message = format!("{error:#}"); + assert!(message.contains("recorded port is in use"), "{message}"); + + // Nothing was spawned and the prior truthful record is untouched: it must + // not have been rewritten as `recovering`. + let on_disk = crate::persistence::load_managed_services(&paths)? + .into_iter() + .find(|candidate| candidate.service_id == "svc-recovery-port-collision") + .expect("record still present"); + fs::remove_dir_all(root).ok(); + drop(occupant); + + assert_eq!(on_disk.status, "failed"); + assert_eq!(on_disk.restart_count, record.restart_count); + Ok(()) + } + + #[test] + #[ignore = "spawned as a child process by terminate_recovery_supervisor_kills_and_reaps_the_child"] + fn recovery_cleanup_sleeper_child() { + if std::env::var_os("ROCMD_TEST_SLEEPER_CHILD").is_none() { + return; + } + thread::sleep(Duration::from_mins(2)); + } + + #[test] + fn terminate_recovery_supervisor_kills_and_reaps_the_child() { + // A real long-lived process stands in for a supervisor that was spawned + // but whose record could not be published. + let mut child = ProcessCommand::new(std::env::current_exe().expect("test binary")) + .arg("watchers::tests::recovery_cleanup_sleeper_child") + .arg("--exact") + .arg("--ignored") + .env("ROCMD_TEST_SLEEPER_CHILD", "1") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn sleeper child"); + let child_pid = child.id(); + + let started = std::time::Instant::now(); + terminate_recovery_supervisor(&mut child); + // The sleeper sleeps for two minutes by itself: only a kill returns + // well before that, so waiting it out must not pass. + assert!( + started.elapsed() < Duration::from_secs(30), + "terminate_recovery_supervisor took {:?}: the child was waited out, not killed", + started.elapsed() + ); + + // The helper reaps, so the pid must be gone rather than a zombie. + assert!( + !rocm_core::process_is_running(child_pid), + "an unrecordable supervisor must be terminated and reaped" + ); + // The helper already reaped it; wait again so the child is provably + // handled on every path in this function (and reaping is idempotent). + assert!(child.wait().is_ok()); + } + + /// If publishing the `recovering` transition fails, nothing has been + /// spawned yet: the rollback at that first write must undo the in-memory + /// transition and leave the on-disk record exactly as it was, not a + /// half-applied `recovering` row. The manifest write goes through + /// [`crate::write_atomic`][rocm_core_write_atomic]-style write-then-rename, + /// so a read-only *file* no longer blocks it (rename ignores the + /// destination's own permission bits) — a read-only *directory* still does, + /// since it can't hold the new temp file the rename needs. + /// + /// [rocm_core_write_atomic]: rocm_core + /// + /// Unix only and skipped as root, where a read-only mode is not enforced + /// and the write would simply succeed. + #[cfg(unix)] + #[test] + fn restart_rolls_back_the_in_memory_record_when_the_first_write_fails() -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + if rocm_core::openmpi::running_as_root() { + return Ok(()); + } + + let (root, paths) = temp_app_paths("restart-rollback-first-write"); + paths.ensure()?; + let service_id = "svc-restart-rollback-first-write"; + let mut record = identity_probe_record(&paths, service_id, free_loopback_port()); + record.status = "failed".to_owned(); + record.write()?; + + let services_dir = paths.services_dir(); + // The launch lock and service log files are both created lazily, on + // first acquire and on first open respectively; pre-create them so + // making the directory read-only below blocks only the manifest's + // write-then-rename, not those earlier opens. + fs::File::create(paths.managed_launch_lock_path())?; + fs::File::create(&record.log_path)?; + fs::set_permissions(&services_dir, std::fs::Permissions::from_mode(0o555))?; + + let result = restart_managed_service(&paths, &mut record); + + // Restore write access so the rest of the write (if any landed) can be + // read back, and so cleanup can remove the directory. + fs::set_permissions(&services_dir, std::fs::Permissions::from_mode(0o755))?; + let persisted = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + let error = result.expect_err("a read-only manifest must fail the first write"); + assert!( + format!("{error:#}").contains("failed to publish the recovering service record"), + "{error:#}" + ); + assert_eq!( + record.status, "failed", + "a failed first write must restore the in-memory record rather than leave it \ + \"recovering\"" + ); + assert_eq!( + persisted?.status, "failed", + "nothing must reach disk when the first write itself fails" + ); + Ok(()) + } + + /// The first-write rollback above covers a failure before anything is + /// spawned; this one covers the second write, after the supervisor exists. + /// The spawn seam corrupts the manifest's content right after the first + /// write (`recovering`, no supervisor yet), so + /// `record_spawned_supervisor`'s read fails. That must kill the spawned + /// supervisor and restore the prior record — and the rollback write itself + /// must still land, since corrupting *content* leaves the file a writable + /// regular file. + /// + /// The spawned supervisor here is a re-exec of this very test binary, + /// which rejects its `supervise` arguments and exits on its own almost + /// immediately regardless of whether anything kills it, so asserting + /// `process_is_running` on it afterward would not catch the kill being + /// dropped. `TERMINATE_RECOVERY_SUPERVISOR_CALLS` asserts the call itself + /// is reached instead; `terminate_recovery_supervisor_kills_and_reaps_the_child` + /// below proves what that call does against a real long-lived process. + #[test] + fn restart_rolls_back_and_kills_the_supervisor_when_the_second_write_cannot_be_read() + -> Result<()> { + let (root, paths) = temp_app_paths("restart-rollback-second-write-unreadable"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let service_id = "svc-restart-rollback-second-write-unreadable"; + let mut record = identity_probe_record(&paths, service_id, free_loopback_port()); + record.status = "failed".to_owned(); + record.write()?; + let prior = record.clone(); + + let terminate_calls_before = TERMINATE_RECOVERY_SUPERVISOR_CALLS.with(std::cell::Cell::get); + let manifest_path = paths.service_manifest_path(service_id); + // The spawn seam runs after the restart's first write and before + // `record_spawned_supervisor`, which is exactly the window to corrupt. + let corrupt = |command: &mut ProcessCommand| -> std::io::Result { + fs::write(&manifest_path, b"not json")?; + command.spawn() + }; + let outcome = restart_managed_service_with(&paths, &mut record, Some(&corrupt)); + let persisted = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + let error = outcome.expect_err("an unreadable second write must fail the restart"); + let message = format!("{error:#}"); + assert!( + message.contains("failed to publish the recovering service record"), + "{message}" + ); + assert!( + !message.contains("rollback also failed"), + "corrupting content, not permissions, leaves the rollback write itself able to \ + land: {message}" + ); + let terminate_calls_after = TERMINATE_RECOVERY_SUPERVISOR_CALLS.with(std::cell::Cell::get); + assert_eq!( + terminate_calls_after, + terminate_calls_before + 1, + "an unreadable second write must still kill the spawned supervisor" + ); + assert_eq!( + record.status, prior.status, + "the in-memory record must be restored to what it was before this restart" + ); + let persisted = persisted?; + assert_eq!( + persisted.status, prior.status, + "the rollback write must overwrite the corrupted content with the prior record" + ); + assert_eq!(persisted.supervisor_pid, prior.supervisor_pid); + Ok(()) + } + + /// As above, but the rollback write itself also fails: a manifest made + /// read-only after the first write still lets `record_spawned_supervisor` + /// *read* it (ownership check passes), but both its write and the + /// subsequent rollback write hit the same permission error. That must be + /// reported as a rollback failure, not swallowed as the original one. + /// + /// The manifest writes go through write-then-rename, so blocking the + /// *directory* (not the file) is what actually denies them. + /// + /// Unix only and skipped as root, where a read-only mode is not enforced. + #[cfg(unix)] + #[test] + fn restart_reports_when_the_rollback_write_also_fails() -> Result<()> { + use std::os::unix::fs::PermissionsExt; + + if rocm_core::openmpi::running_as_root() { + return Ok(()); + } + + let (root, paths) = temp_app_paths("restart-rollback-also-fails"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let service_id = "svc-restart-rollback-also-fails"; + let mut record = identity_probe_record(&paths, service_id, free_loopback_port()); + record.status = "failed".to_owned(); + record.write()?; + + let services_dir = paths.services_dir(); + // Same seam as above: deny directory writes once the first write has + // landed, so the second write and its rollback both fail. + let lock = |command: &mut ProcessCommand| -> std::io::Result { + fs::set_permissions(&services_dir, std::fs::Permissions::from_mode(0o555))?; + command.spawn() + }; + let outcome = restart_managed_service_with(&paths, &mut record, Some(&lock)); + fs::set_permissions(&services_dir, std::fs::Permissions::from_mode(0o755))?; + let persisted = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + let error = outcome.expect_err("a read-only manifest must fail both writes"); + let message = format!("{error:#}"); + assert!( + message.contains("rollback also failed"), + "both the second write and its rollback hit the same permission error: {message}" + ); + assert!( + message.contains(service_id), + "the rollback-also-failed message must name the affected service: {message}" + ); + // Left exactly as the second write's failure found it: `recovering` + // with no supervisor, matching the "no supervisor" the error names. + assert_eq!(persisted?.status, "recovering"); + Ok(()) + } + + /// A spawn that fails after the `recovering` write must not leave that + /// write behind: `is_live` counts `recovering`, and rocmd reads records + /// without refreshing them, so a stale one would reserve the port. + #[test] + fn restart_rolls_back_the_recovering_record_when_the_spawn_fails() -> Result<()> { + let (root, paths) = temp_app_paths("restart-spawn-fails-rolls-back"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let service_id = "svc-restart-spawn-fails"; + let mut record = identity_probe_record(&paths, service_id, free_loopback_port()); + record.status = "failed".to_owned(); + record.write()?; + + let failing = |_command: &mut ProcessCommand| -> std::io::Result { + Err(std::io::Error::other("spawn refused by test")) + }; + let outcome = restart_managed_service_with(&paths, &mut record, Some(&failing)); + let persisted = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + let message = format!( + "{:#}", + outcome.expect_err("a failed spawn fails the restart") + ); + assert!( + message.contains("failed to spawn recovery supervisor"), + "{message}" + ); + assert_eq!( + persisted?.status, "failed", + "the on-disk record must not stay `recovering`" + ); + assert_eq!( + record.status, "failed", + "the in-memory record must be restored too" + ); + Ok(()) + } + + /// The disk re-check excludes the service's own record: rocmd reads records + /// as-is, so a crashed service can still read `running` on its own port, and + /// counting that as a competitor would make it impossible to ever recover. + #[test] + fn recovery_ignores_its_own_live_looking_record_on_its_own_port() -> Result<()> { + use std::net::TcpListener; + + let (root, paths) = temp_app_paths("recovery-self-exclusion"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let probe = TcpListener::bind(("127.0.0.1", 0))?; + let port = probe.local_addr()?.port(); + drop(probe); + + let service_id = "svc-recovery-self-exclusion"; + let mut record = identity_probe_record(&paths, service_id, port); + record.status = "running".to_owned(); + record.write()?; + + let failing = |_command: &mut ProcessCommand| -> std::io::Result { + Err(std::io::Error::other("spawn refused by test")) + }; + let outcome = restart_managed_service_with(&paths, &mut record, Some(&failing)); + fs::remove_dir_all(root).ok(); + + let message = format!("{:#}", outcome.expect_err("the intercepted spawn fails")); + assert!( + message.contains("spawn refused by test"), + "recovery must reach the spawn, not refuse over its own record: {message}" + ); + Ok(()) + } + + /// A `rocm services restart` that finished while recovery waited for the + /// launch lock changed this service's own record; recovery must refuse + /// rather than write its stale copy over it. + #[test] + fn recovery_refuses_when_its_own_record_changed_while_waiting_for_the_lock() -> Result<()> { + let (root, paths) = temp_app_paths("recovery-stale-own-record"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let service_id = "svc-recovery-stale-own-record"; + let mut record = identity_probe_record(&paths, service_id, free_loopback_port()); + record.status = "failed".to_owned(); + record.write()?; + let mut stale = record.clone(); + + // The restart that won the lock: same id, now starting under new pids. + record.status = "starting".to_owned(); + record.supervisor_pid = 4_242_424; + record.write()?; + + let never = |_command: &mut ProcessCommand| -> std::io::Result { + panic!("recovery must not spawn over a restarted record") + }; + let outcome = restart_managed_service_with(&paths, &mut stale, Some(&never)); + let persisted = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + let message = format!( + "{:#}", + outcome.expect_err("a changed record refuses recovery") + ); + assert!(message.contains("record changed"), "{message}"); + let persisted = persisted?; + assert_eq!(persisted.status, "starting"); + assert_eq!(persisted.supervisor_pid, 4_242_424); + Ok(()) + } + + /// A stop requested while recovery waited for the lock must not be cleared. + #[test] + fn recovery_refuses_when_a_stop_was_requested_while_waiting_for_the_lock() -> Result<()> { + let (root, paths) = temp_app_paths("recovery-stop-requested"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let service_id = "svc-recovery-stop-requested"; + let mut record = identity_probe_record(&paths, service_id, free_loopback_port()); + record.status = "failed".to_owned(); + record.write()?; + let mut stale = record.clone(); + + record.stop_requested_unix_ms = Some(unix_time_millis()); + record.write()?; + + let never = |_command: &mut ProcessCommand| -> std::io::Result { + panic!("recovery must not spawn for a service being stopped") + }; + let outcome = restart_managed_service_with(&paths, &mut stale, Some(&never)); + let persisted = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + let message = format!( + "{:#}", + outcome.expect_err("a pending stop refuses recovery") + ); + assert!(message.contains("stop was requested"), "{message}"); + assert!(stop_requested(&persisted?)); + Ok(()) + } + + /// A manifest deleted while recovery waited must not be resurrected. + #[test] + fn recovery_refuses_when_the_record_was_removed_while_waiting_for_the_lock() -> Result<()> { + let (root, paths) = temp_app_paths("recovery-removed-record"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let service_id = "svc-recovery-removed-record"; + let mut record = identity_probe_record(&paths, service_id, free_loopback_port()); + record.status = "failed".to_owned(); + record.write()?; + let mut stale = record; + fs::remove_file(paths.service_manifest_path(service_id))?; + + let never = |_command: &mut ProcessCommand| -> std::io::Result { + panic!("recovery must not spawn for a removed service") + }; + let outcome = restart_managed_service_with(&paths, &mut stale, Some(&never)); + let resurrected = paths.service_manifest_path(service_id).exists(); + fs::remove_dir_all(root).ok(); + + let message = format!( + "{:#}", + outcome.expect_err("a removed record refuses recovery") + ); + assert!(message.contains("was removed"), "{message}"); + assert!(!resurrected, "recovery must not rewrite a deleted manifest"); + Ok(()) + } + + /// Recovery waits a bounded time for the launch lock, then gives up instead + /// of queuing behind an interactive launch. + #[test] + fn recovery_gives_up_when_the_launch_lock_stays_held_past_its_bound() -> Result<()> { + let (root, paths) = temp_app_paths("recovery-lock-bound"); + paths.ensure()?; + let mut record = identity_probe_record(&paths, "svc-lock-bound", free_loopback_port()); + record.status = "failed".to_owned(); + record.write()?; + + let guard = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; + let holder = thread::spawn(move || { + thread::sleep(Duration::from_secs(3)); + drop(guard); + }); + let started = std::time::Instant::now(); + let outcome = + restart_managed_service_bounded(&paths, &mut record, None, Duration::from_millis(100)); + let waited = started.elapsed(); + holder.join().expect("holder thread"); + fs::remove_dir_all(root).ok(); + + assert!(outcome.is_err(), "a held lock must defer recovery"); + assert!( + waited < Duration::from_secs(2), + "recovery waited {waited:?} for a lock held for 3s: the wait is not bounded" + ); + Ok(()) + } + + /// A candidate still inside its restart backoff must not starve the ones + /// after it. + #[test] + fn find_recoverable_service_skips_a_candidate_inside_its_backoff() -> Result<()> { + let (root, paths) = temp_app_paths("recoverable-backoff-skip"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let b_created = unix_time_millis() - 10_000; + let mut a = identity_probe_record(&paths, "svc-backoff-a", free_loopback_port()); + a.status = "failed".to_owned(); + a.last_restart_unix_ms = Some(unix_time_millis()); + // Records load newest first: A must come first, or the skip is not what + // gets B returned. + a.created_at_unix_ms = b_created + 1_000; + a.write()?; + let mut b = identity_probe_record(&paths, "svc-backoff-b", free_loopback_port()); + b.status = "failed".to_owned(); + b.created_at_unix_ms = b_created; + b.write()?; + + let found = find_recoverable_service(&paths); + fs::remove_dir_all(root).ok(); + + let (record, _reason) = found?.expect("service B is recoverable"); + assert_eq!(record.service_id, "svc-backoff-b"); + Ok(()) + } + + /// The rollback restores the prior record when handed the snapshot taken + /// after the supervisor was recorded, which is the one the `try_wait` + /// failure arm passes. That arm cannot be driven from here, so this covers + /// the rollback's guard, not the choice of snapshot at that call site. + #[test] + fn rollback_against_the_recorded_snapshot_restores_the_prior_record() -> Result<()> { + let (root, paths) = temp_app_paths("rollback-recorded-snapshot"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let mut prior = identity_probe_record(&paths, "svc-rollback-recorded", 11466); + prior.status = "failed".to_owned(); + let mut recorded = prior.clone(); + recorded.status = "recovering".to_owned(); + recorded.supervisor_pid = 4_242_424; + recorded.write()?; + + rollback_recovering_record(&paths, &recorded, &prior)?; + let persisted = load_service_record(&paths, "svc-rollback-recorded"); + fs::remove_dir_all(root).ok(); + + assert_eq!(persisted?.status, "failed"); + Ok(()) + } + + /// The rollback merges onto the live record: a stop marker persisted since + /// the first write survives, and the restart's own fields are restored. + #[test] + fn rollback_keeps_a_concurrent_stop_marker_and_restores_the_restarts_fields() -> Result<()> { + let (root, paths) = temp_app_paths("rollback-merge"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let mut prior = identity_probe_record(&paths, "svc-rollback-merge", 11464); + prior.status = "failed".to_owned(); + // Distinct values, so a restore that is skipped shows up. + prior.restart_count = 7; + prior.engine_pid = Some(4_111_111); + prior.inference_verified_at_unix_ms = Some(5_555); + let mut recovering = prior.clone(); + recovering.status = "recovering".to_owned(); + recovering.supervisor_pid = 0; + recovering.restart_count = 8; + recovering.engine_pid = None; + recovering.inference_verified_at_unix_ms = None; + // What a concurrent stop persisted on the live `recovering` record. + let mut on_disk = recovering.clone(); + on_disk.stop_requested_unix_ms = Some(1_234); + on_disk.write()?; + + rollback_recovering_record(&paths, &recovering, &prior)?; + let persisted = load_service_record(&paths, "svc-rollback-merge"); + fs::remove_dir_all(root).ok(); + + let persisted = persisted?; + assert_eq!(persisted.status, "failed"); + assert_eq!(persisted.restart_count, 7); + assert_eq!(persisted.engine_pid, Some(4_111_111)); + assert_eq!(persisted.inference_verified_at_unix_ms, Some(5_555)); + assert_eq!( + persisted.stop_requested_unix_ms, + Some(1_234), + "a stop marker persisted meanwhile must survive the rollback" + ); + Ok(()) + } + + /// If another writer has since recorded a different supervisor, that + /// record is truer than this restart's view and the rollback leaves it. + #[test] + fn rollback_leaves_a_record_that_names_a_different_supervisor() -> Result<()> { + let (root, paths) = temp_app_paths("rollback-other-supervisor"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let mut prior = identity_probe_record(&paths, "svc-rollback-other", 11465); + prior.status = "failed".to_owned(); + let mut recovering = prior.clone(); + recovering.status = "recovering".to_owned(); + recovering.supervisor_pid = 0; + let mut on_disk = recovering.clone(); + on_disk.supervisor_pid = 4_242_424; + on_disk.write()?; + + rollback_recovering_record(&paths, &recovering, &prior)?; + let persisted = load_service_record(&paths, "svc-rollback-other"); + fs::remove_dir_all(root).ok(); + + let persisted = persisted?; + assert_eq!( + persisted.status, "recovering", + "must not be blindly restored" + ); + assert_eq!(persisted.supervisor_pid, 4_242_424); + Ok(()) + } + + /// The lease branch above proves a bound loopback port refuses recovery; + /// the doc comment on `restart_managed_service`'s lease also claims custom + /// hosts skip that preflight entirely, relying on the engine's own bind + /// error instead. Binding the exact recorded port on a custom host and + /// still reaching a spawned supervisor is what proves that claim, rather + /// than just trusting the comment. + /// + /// Linux-only and ignores the restart's own outcome for the same reason as + /// `restart_managed_service_persists_the_supervisor_identity_token`: the + /// restart re-execs the test binary, which rejects the `supervise` + /// arguments and exits at once, so the restart itself still reports + /// failure — after writing the supervisor identity this test reads. + #[cfg(target_os = "linux")] + #[test] + fn restart_on_a_custom_host_skips_the_loopback_lease() -> Result<()> { + let (root, paths) = temp_app_paths("restart-custom-host-skips-lease"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let service_id = "svc-restart-custom-host"; + let blocked_port = free_loopback_port(); + let mut record = ManagedServiceRecord::new( + &paths, + service_id, + "llamacpp", + "a-model", + "a-model", + "0.0.0.0", + blocked_port, + "managed", + std::process::id(), + None, + None, + Some("gpu_required".to_owned()), + ); + record.status = "failed".to_owned(); + record.write()?; + + let _blocker = std::net::TcpListener::bind(("0.0.0.0", blocked_port))?; + + let outcome = restart_managed_service(&paths, &mut record); + let persisted = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + let message = outcome.err().map(|e| format!("{e:#}")).unwrap_or_default(); + assert!( + !message.contains("its recorded port is in use"), + "a custom host must not preflight the port with a loopback lease: {message}" + ); + assert!( + persisted?.supervisor_start_ticks.is_some(), + "recovery must still reach a spawned supervisor despite the bound port" + ); + Ok(()) + } + /// A stop that lands while a restart waits to see whether its supervisor /// survived startup must outlive the restart's `failed` write: a `failed` /// record without the marker is exactly what recovery restarts. diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index acc533d54..9202ca6b8 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -153,7 +153,10 @@ pub use proc_lifecycle::{ process_start_ticks, terminate_verified, }; pub use process::{ - FileLock, detach_command_session, process_is_running, terminate_process, terminate_process_tree, + FileLock, LoopbackPortLease, MANAGED_LAUNCH_LOCK_TIMEOUT, detach_command_session, + is_canonical_loopback_host, launch_port_argument, lease_loopback_port, + lease_loopback_port_or_bind_error, process_is_running, terminate_process, + terminate_process_tree, }; #[cfg(windows)] pub use process::{ diff --git a/crates/rocm-core/src/process.rs b/crates/rocm-core/src/process.rs index afbea2860..8134df80c 100644 --- a/crates/rocm-core/src/process.rs +++ b/crates/rocm-core/src/process.rs @@ -24,12 +24,14 @@ //! capture helpers onto it is tracked in //! [#623](https://github.com/ROCm/rocm-cli/issues/623). +use crate::DEFAULT_LOCAL_HOST; use anyhow::{Context, Result, bail}; #[cfg(windows)] use std::collections::BTreeMap; #[cfg(windows)] use std::ffi::{OsStr, OsString}; use std::fs; +use std::net::TcpListener; #[cfg(windows)] use std::os::windows::ffi::OsStrExt; use std::path::{Path, PathBuf}; @@ -403,13 +405,8 @@ pub struct FileLock { } impl FileLock { - /// Acquire an exclusive lock on `path`, creating the lock file and any - /// missing parent directories first. Blocks until the lock is available. - /// - /// The lock file itself carries no data; it is a rendezvous point, so an - /// existing file is reused (never truncated) and its contents are ignored. - pub fn acquire(path: impl Into) -> Result { - let path = path.into(); + /// Create (never truncate) the lock file and any missing parent directories. + fn open(path: PathBuf) -> Result<(PathBuf, fs::File)> { if let Some(parent) = path.parent() { fs::create_dir_all(parent) .with_context(|| format!("failed to create lock directory {}", parent.display()))?; @@ -421,6 +418,16 @@ impl FileLock { .truncate(false) .open(&path) .with_context(|| format!("failed to open lock file {}", path.display()))?; + Ok((path, file)) + } + + /// Acquire an exclusive lock on `path`, creating the lock file and any + /// missing parent directories first. Blocks until the lock is available. + /// + /// The lock file itself carries no data; it is a rendezvous point, so an + /// existing file is reused (never truncated) and its contents are ignored. + pub fn acquire(path: impl Into) -> Result { + let (path, file) = Self::open(path.into())?; file.lock() .with_context(|| format!("failed to acquire lock {}", path.display()))?; Ok(Self { file, path }) @@ -431,6 +438,201 @@ impl FileLock { pub fn path(&self) -> &Path { &self.path } + + /// Like [`Self::acquire`], but fails with an actionable error instead of + /// blocking forever once `timeout` has elapsed. + /// + /// Used where an indefinite wait would be a worse failure mode than a + /// clear timeout message — e.g. `rocmd`'s automatic recovery, which must + /// not hang forever behind a `rocm serve` launch holding the same lock. + /// A `rocm serve` launch itself uses the unbounded [`Self::acquire`]: its + /// own worst-case hold time (the already-running probe inside the launch + /// transaction) can exceed [`MANAGED_LAUNCH_LOCK_TIMEOUT`], so bounding it + /// would trade a real, slow-but-working launch for a spurious failure. + /// + /// # Errors + /// Fails when the lock file cannot be created/locked, or when another + /// holder keeps it for the whole timeout. + pub fn acquire_bounded(path: impl Into, timeout: Duration) -> Result { + let (path, file) = Self::open(path.into())?; + let deadline = Instant::now() + timeout; + loop { + match file.try_lock() { + Ok(()) => return Ok(Self { file, path }), + Err(std::fs::TryLockError::WouldBlock) => {} + Err(std::fs::TryLockError::Error(error)) => { + return Err(anyhow::Error::new(error) + .context(format!("failed to lock {}", path.display()))); + } + } + let now = Instant::now(); + if now >= deadline { + bail!( + "timed out after {timeout:?} waiting for the lock at {path}; another \ + `rocm serve` launch or `rocmd` recovery is mid-transaction. Retry in a \ + moment, and check `rocm services` for a stuck launch.", + path = path.display() + ); + } + thread::sleep(FILE_LOCK_BOUNDED_POLL.min(deadline - now)); + } + } +} + +/// Polling interval used while waiting on [`FileLock::acquire_bounded`]. +const FILE_LOCK_BOUNDED_POLL: Duration = Duration::from_millis(25); + +/// Total time `rocmd`'s automatic recovery waits for the launch lock. +/// +/// Bounds contention on [`crate::paths::AppPaths::managed_launch_lock_path`] during the +/// port-allocation window, failing with an actionable error instead of hanging. Only +/// `rocmd`'s recovery/restart path is bounded this way; a `rocm serve` launch holds the +/// same lock with the unbounded [`FileLock::acquire`] (see its doc comment on +/// [`FileLock::acquire_bounded`] for why). +pub const MANAGED_LAUNCH_LOCK_TIMEOUT: Duration = Duration::from_secs(5); + +// --------------------------------------------------------------------------- +// Managed-service port leasing. +// +// Both `rocm serve` and `rocmd` recovery contend for the same local TCP ports +// and the same service records, so the primitives that arbitrate them live here +// rather than in either binary. The transaction itself is serialized by the +// existing [`crate::paths::AppPaths::managed_launch_lock_path`] / [`FileLock`] — see +// [`FileLock::acquire_bounded`] for the bounded variant this contention uses. +// --------------------------------------------------------------------------- + +/// True only for the exact canonical default loopback host. +/// +/// Proactive OS collision detection (leasing a real socket before launch) is +/// claimed only for this one host. `localhost`, `::1`, `0.0.0.0`, and other +/// IPv4 addresses are "custom hosts": they require an explicit port and keep the +/// existing engine-owned bind/error behavior, because an IPv4-loopback probe +/// would not prove anything about the address the engine will actually bind. +#[must_use] +pub fn is_canonical_loopback_host(host: &str) -> bool { + host.trim() == DEFAULT_LOCAL_HOST +} + +/// Validate the optional `port` argument of a `launch_server` tool call. +/// +/// `None` and JSON `null` mean "not supplied" (automatic selection). Anything +/// else must be an integer in 1-65535: a malformed value must not be read as +/// "not supplied" and silently fall through to automatic port selection. +/// Shared by the chat assistant and the MCP server so the two cannot drift. +/// +/// # Errors +/// Returns the reason, to be prefixed with the caller's own argument name. +pub fn launch_port_argument( + value: Option<&serde_json::Value>, +) -> Result, &'static str> { + let Some(value) = value.filter(|value| !value.is_null()) else { + return Ok(None); + }; + let port = value.as_u64().ok_or("must be an integer")?; + match u16::try_from(port) { + Ok(port) if port != 0 => Ok(Some(port)), + _ => Err("must be between 1 and 65535"), + } +} + +/// A real, held `127.0.0.1:` listener proving the port is bindable now. +/// +/// The lease is the collision check: it is acquired before any manifest, log, or +/// child process exists, and [`LoopbackPortLease::release`] drops the socket +/// immediately before the engine child is spawned so the child can bind it. The +/// window between release and the child's bind is unavoidable without engine +/// socket-passing; it is far narrower than the previous "bind and hope" launch. +#[derive(Debug)] +pub struct LoopbackPortLease { + port: u16, + listener: Option, +} + +impl LoopbackPortLease { + #[must_use] + pub const fn port(&self) -> u16 { + self.port + } + + /// True while this lease still holds the socket open. + /// + /// No production caller needs this: `release` is the only thing that ever + /// changes it, and callers already know whether they called `release`. It + /// stays `pub` because downstream crates' tests assert on it directly + /// (e.g. `apps/rocm`'s `serve_port` tests), where `#[cfg(test)]` here + /// would not apply. + #[must_use] + pub const fn is_held(&self) -> bool { + self.listener.is_some() + } + + /// Close the socket and hand the port to the caller. Call this immediately + /// before spawning the process that will bind it. + pub fn release(mut self) -> u16 { + self.listener = None; + self.port + } + + /// A lease over a port this process never probed, for custom hosts where the + /// engine, not the CLI, owns bind success or failure. + #[must_use] + pub const fn engine_owned(port: u16) -> Self { + Self { + port, + listener: None, + } + } +} + +/// Try to take an exclusive IPv4-loopback lease on `port`. +/// +/// Errors are returned verbatim so callers can distinguish "occupied" +/// ([`std::io::ErrorKind::AddrInUse`], the only condition that should advance an +/// automatic scan) from a privileged or unavailable address, which must fail +/// immediately rather than silently choosing a different port. +pub fn lease_loopback_port(port: u16) -> std::io::Result { + let listener = TcpListener::bind(std::net::SocketAddrV4::new( + std::net::Ipv4Addr::LOCALHOST, + port, + ))?; + Ok(LoopbackPortLease { + port, + listener: Some(listener), + }) +} + +/// [`lease_loopback_port`], but mapped to an `anyhow::Error`. +/// +/// Only uses `in_use_message` for a genuine collision +/// ([`std::io::ErrorKind::AddrInUse`]). Any other bind failure (permission +/// denied, address unavailable, ...) gets its own generic message instead — +/// mislabelling those as "in use" sends the user to stop a service that +/// isn't the actual problem. Mirrors `rocm serve`'s own +/// `explicit_bind_error`; shared by the CLI's restart re-check and +/// `rocmd`'s recovery re-check so neither duplicates (or drifts from) that +/// distinction. +pub fn lease_loopback_port_or_bind_error( + port: u16, + in_use_message: impl std::fmt::Display, +) -> anyhow::Result { + lease_loopback_port(port).map_err(|error| bind_error(port, &error, in_use_message)) +} + +/// Pure error-mapping half of [`lease_loopback_port_or_bind_error`], split out +/// so the `AddrInUse`-vs-other-error distinction is tested against a +/// synthetic `io::Error` rather than needing a real bind failure (a +/// privileged port, an unavailable address) that is not reliably +/// reproducible in CI. +fn bind_error( + port: u16, + error: &std::io::Error, + in_use_message: impl std::fmt::Display, +) -> anyhow::Error { + if error.kind() == std::io::ErrorKind::AddrInUse { + anyhow::anyhow!("{in_use_message}") + } else { + anyhow::anyhow!("failed to reserve port {port}: {error}") + } } impl Drop for FileLock { @@ -1018,6 +1220,207 @@ fn assemble_output( #[cfg(test)] mod tests { use super::*; + use crate::test_support::workspace_test_artifact_dir; + use std::process::Stdio; + + const LOCK_CHILD_PATH_ENV: &str = "ROCM_CORE_TEST_LOCK_PATH"; + const LOCK_CHILD_READY_ENV: &str = "ROCM_CORE_TEST_LOCK_READY"; + const HANDOFF_PORT_ENV: &str = "ROCM_CORE_TEST_HANDOFF_PORT"; + + fn lock_test_dir(label: &str) -> PathBuf { + let dir = workspace_test_artifact_dir().join(format!( + "{label}-{}-{}", + std::process::id(), + crate::unix_time_millis() + )); + fs::create_dir_all(&dir).expect("create lock test dir"); + dir + } + + /// Re-run this test binary, executing exactly one `#[ignore]`d helper test in + /// a genuinely separate process. Cross-process locking and socket handoff + /// cannot be proven inside one process. + fn spawn_helper_test(name: &str, envs: &[(&str, String)]) -> std::process::Child { + let exe = std::env::current_exe().expect("test binary path"); + let mut command = Command::new(exe); + command + .arg(name) + .arg("--exact") + .arg("--ignored") + .arg("--test-threads") + .arg("1") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + for (key, value) in envs { + command.env(key, value); + } + command.spawn().expect("spawn helper test process") + } + + #[test] + fn canonical_loopback_host_is_only_the_exact_default_address() { + assert!(is_canonical_loopback_host(DEFAULT_LOCAL_HOST)); + assert!(is_canonical_loopback_host(" 127.0.0.1 ")); + // Everything else is a "custom host": explicit port, engine-owned bind. + for host in [ + "localhost", + "::1", + "0.0.0.0", + "127.0.0.2", + "192.168.1.4", + "", + ] { + assert!( + !is_canonical_loopback_host(host), + "{host} must not be treated as the canonical loopback host" + ); + } + } + + #[test] + fn bounded_file_lock_excludes_independent_handles_and_fails_closed_on_timeout() { + let path = lock_test_dir("core-bounded-lock-exclusive").join("launch.lock"); + let held = FileLock::acquire_bounded(&path, Duration::from_secs(5)).expect("first holder"); + assert_eq!(held.path(), path.as_path()); + + // A second, independent file handle must contend on the OS lock rather + // than sail through, and must give up with an actionable error. + let started = Instant::now(); + let error = FileLock::acquire_bounded(&path, Duration::from_millis(200)) + .expect_err("a held lock must not be acquired twice"); + assert!( + started.elapsed() >= Duration::from_millis(200), + "waiter returned before its deadline" + ); + let message = format!("{error:#}"); + assert!(message.contains("timed out"), "{message}"); + + drop(held); + FileLock::acquire_bounded(&path, Duration::from_secs(5)) + .expect("lock is free once the guard is dropped"); + } + + #[test] + #[ignore = "spawned as a child process by bounded_file_lock_is_released_when_the_holder_process_exits"] + fn bounded_file_lock_child_holder() { + let Ok(path) = std::env::var(LOCK_CHILD_PATH_ENV) else { + return; + }; + let ready = std::env::var(LOCK_CHILD_READY_ENV).expect("ready-file path"); + let guard = FileLock::acquire_bounded(Path::new(&path), Duration::from_secs(10)) + .expect("child acquires the shared lock"); + fs::write(&ready, "held").expect("signal that the child holds the lock"); + thread::sleep(Duration::from_millis(300)); + // Leak the guard so the lock is released by process exit alone — that is + // the property a crashed launcher depends on. + std::mem::forget(guard); + } + + #[test] + fn bounded_file_lock_is_released_when_the_holder_process_exits() { + let dir = lock_test_dir("core-bounded-lock-crossproc"); + let path = dir.join("launch.lock"); + let ready = dir.join("child-ready"); + let mut child = spawn_helper_test( + "process::tests::bounded_file_lock_child_holder", + &[ + (LOCK_CHILD_PATH_ENV, path.display().to_string()), + (LOCK_CHILD_READY_ENV, ready.display().to_string()), + ], + ); + + let deadline = Instant::now() + Duration::from_mins(1); + while !ready.exists() { + assert!( + Instant::now() < deadline, + "child never reported holding the lock" + ); + thread::sleep(Duration::from_millis(25)); + } + + let busy = FileLock::acquire_bounded(&path, Duration::from_millis(50)) + .expect_err("the lock must be exclusive across processes"); + assert!(format!("{busy:#}").contains("timed out")); + + let status = child.wait().expect("child holder exits"); + assert!(status.success(), "child holder failed: {status}"); + FileLock::acquire_bounded(&path, Duration::from_secs(10)) + .expect("the OS releases the lock when the holding process exits"); + } + + #[test] + fn loopback_lease_blocks_a_second_bind_until_released() { + let ephemeral = TcpListener::bind(("127.0.0.1", 0)).expect("ephemeral bind"); + let port = ephemeral.local_addr().expect("local addr").port(); + drop(ephemeral); + + let lease = lease_loopback_port(port).expect("free port leases"); + assert_eq!(lease.port(), port); + assert!(lease.is_held()); + let conflict = lease_loopback_port(port).expect_err("a held port must not lease twice"); + assert_eq!(conflict.kind(), std::io::ErrorKind::AddrInUse); + + assert_eq!(lease.release(), port); + lease_loopback_port(port).expect("the port is bindable again after release"); + } + + #[test] + fn bind_error_labels_only_a_genuine_collision_as_in_use() { + let in_use = bind_error( + 9000, + &std::io::Error::from(std::io::ErrorKind::AddrInUse), + "its port is in use", + ); + assert_eq!(in_use.to_string(), "its port is in use"); + + let other = bind_error( + 9000, + &std::io::Error::from(std::io::ErrorKind::PermissionDenied), + "its port is in use", + ); + assert_ne!(other.to_string(), "its port is in use"); + assert!(other.to_string().contains("9000")); + } + + #[test] + fn engine_owned_lease_holds_no_socket() { + // Custom hosts get a port carrier, never a loopback probe. + let lease = LoopbackPortLease::engine_owned(8000); + assert_eq!(lease.port(), 8000); + assert!(!lease.is_held()); + } + + #[test] + #[ignore = "spawned as a child process by released_loopback_lease_is_immediately_bindable_by_another_process"] + fn loopback_handoff_child() { + let Ok(port) = std::env::var(HANDOFF_PORT_ENV) else { + return; + }; + let port: u16 = port.parse().expect("handoff port"); + let lease = lease_loopback_port(port).expect("child binds the released port"); + assert_eq!(lease.port(), port); + } + + #[test] + fn released_loopback_lease_is_immediately_bindable_by_another_process() { + let ephemeral = TcpListener::bind(("127.0.0.1", 0)).expect("ephemeral bind"); + let port = ephemeral.local_addr().expect("local addr").port(); + drop(ephemeral); + let lease = lease_loopback_port(port).expect("parent leases the port"); + assert_eq!(lease.release(), port); + + let status = spawn_helper_test( + "process::tests::loopback_handoff_child", + &[(HANDOFF_PORT_ENV, port.to_string())], + ) + .wait() + .expect("handoff child exits"); + assert!( + status.success(), + "a separate process must bind the port immediately after the lease is released" + ); + } #[test] fn file_lock_creates_missing_parent_dirs_and_lock_file() { @@ -2023,3 +2426,45 @@ mod tests { assert_the_child_is_given_an_empty_stdin(fixture.command()); } } + +#[cfg(test)] +mod launch_port_argument_tests { + use super::launch_port_argument; + use serde_json::json; + + #[test] + fn absent_or_null_means_automatic() { + assert_eq!(launch_port_argument(None), Ok(None)); + assert_eq!(launch_port_argument(Some(&json!(null))), Ok(None)); + } + + #[test] + fn accepts_the_full_valid_range() { + assert_eq!(launch_port_argument(Some(&json!(1))), Ok(Some(1))); + assert_eq!(launch_port_argument(Some(&json!(65535))), Ok(Some(65535))); + } + + #[test] + fn rejects_malformed_values_instead_of_treating_them_as_absent() { + for value in [ + json!("8000"), + json!(8000.5), + json!(-1), + json!(true), + json!([8000]), + ] { + assert_eq!( + launch_port_argument(Some(&value)), + Err("must be an integer"), + "{value}" + ); + } + for value in [json!(0), json!(65536), json!(u64::MAX)] { + assert_eq!( + launch_port_argument(Some(&value)), + Err("must be between 1 and 65535"), + "{value}" + ); + } + } +} diff --git a/crates/rocm-core/src/service_record.rs b/crates/rocm-core/src/service_record.rs index 37bb07d94..c5022d70b 100644 --- a/crates/rocm-core/src/service_record.rs +++ b/crates/rocm-core/src/service_record.rs @@ -363,6 +363,38 @@ impl ManagedServiceRecord { Ok(self.status != previous) } + /// Whether this record still reserves its port: the sole definition of + /// "live" shared between the CLI's launch-time port check and `rocmd`'s + /// recovery. Same predicate, not same data: the CLI's `load_managed_services` + /// refreshes each record's PID/endpoint liveness before this runs, while + /// `rocmd::persistence::load_managed_services` reads the file as-is. A + /// record whose process has died since its last write can therefore still + /// read `is_live()` to `rocmd` after the CLI would have refreshed it to + /// `"stopped"`. That fails safe — recovery defers rather than racing a + /// collision — but it can stall on a stale record until something else + /// rewrites it. + #[must_use] + pub fn is_live(&self) -> bool { + matches!( + self.status.as_str(), + "ready" | "running" | "starting" | "recovering" + ) + } + + /// Whether this live record reserves `port` — the sole collision check + /// shared by the CLI's restart re-check and `rocmd`'s recovery re-check. + /// See [`Self::is_live`]'s doc for the data-freshness gap between the two + /// callers. + /// + /// Deliberately host-blind: a `0.0.0.0` (or `localhost`) listener also + /// blocks the loopback address a competing bind would target, so comparing + /// hosts would under-detect a real collision rather than merely cost one + /// skipped candidate. + #[must_use] + pub fn blocks_port(&self, port: u16) -> bool { + self.is_live() && self.port == port + } + fn with_storage_paths(&self) -> Self { let mut record = self.clone(); record.manifest_path = normalize_runtime_path_for_storage(&record.manifest_path); @@ -420,6 +452,22 @@ mod tests { assert!(record.last_restart_unix_ms.is_some()); } + #[test] + fn blocks_port_needs_a_live_record_on_that_exact_port() { + let mut record = probe_test_record(11435); + record.status = "running".to_owned(); + assert!(record.blocks_port(11435)); + assert!( + !record.blocks_port(11436), + "a different port is not blocked" + ); + record.status = "stopped".to_owned(); + assert!( + !record.blocks_port(11435), + "a stopped record blocks nothing" + ); + } + /// Writes `state` as the record's engine state file and runs the refresh. fn refresh_from_state(name: &str, state: &serde_json::Value) -> ManagedServiceRecord { let dir = std::env::temp_dir().join(format!( diff --git a/crates/rocm-dash-tui/src/agent/tools.rs b/crates/rocm-dash-tui/src/agent/tools.rs index 0416d26b3..2f169032d 100644 --- a/crates/rocm-dash-tui/src/agent/tools.rs +++ b/crates/rocm-dash-tui/src/agent/tools.rs @@ -592,7 +592,7 @@ rocm_mutating_tool!( "model": { "type": "string", "description": "Model id/name to serve." }, "engine": { "type": "string", "description": "Optional engine, e.g. 'vllm'." }, "host": { "type": "string", "description": "Loopback host only (e.g. 127.0.0.1)." }, - "port": { "type": "integer", "description": "Optional TCP port." }, + "port": { "type": "integer", "description": "TCP port. Required unless host is exactly 127.0.0.1 (localhost/::1 still need one)." }, "device": { "type": "string", "description": "GPU device selector (CPU is rejected)." } }, "required": ["model"] diff --git a/crates/rocm-dash-tui/src/app/mod.rs b/crates/rocm-dash-tui/src/app/mod.rs index 88d483a1e..fad07882d 100644 --- a/crates/rocm-dash-tui/src/app/mod.rs +++ b/crates/rocm-dash-tui/src/app/mod.rs @@ -937,7 +937,8 @@ impl AppState { /// (and is ignored while a job runs) before it can eject the manager. pub(crate) fn active_overlay_at_root(&self) -> bool { self.serve_wizard.as_ref().is_none_or(|w| { - w.browser.is_none() + w.editor.is_none() + && w.browser.is_none() && w.picker.is_none() && w.approval.is_none() && w.active_job.is_none() @@ -1130,7 +1131,7 @@ impl AppState { self.chat_detect_dispatch = true; if self.chat_consent == ChatConsent::Accepted { self.chat.push(ChatTurn::agent( - "Detecting a local engine (Lemonade :13305 / vLLM :8000 / rocm serve :11435)…" + "Detecting a local engine (Lemonade :13305 / vLLM :8000 / a rocm serve service)…" .to_string(), )); } @@ -1160,7 +1161,7 @@ impl AppState { self.chat_detect_offer = Some(cfg); } else { self.chat_detect_offer = None; - let msg = "no local engine found (Lemonade :13305 / vLLM :8000 / rocm serve :11435)"; + let msg = "no local engine found (Lemonade :13305 / vLLM :8000 / a rocm serve service)"; if accepted { self.chat.push(ChatTurn::agent(msg.to_string())); } else { @@ -1864,6 +1865,56 @@ mod tests { ); } + #[test] + fn serve_wizard_editor_handles_escape_before_overlay_back_out() { + let mut state = AppState::new("t".into(), "default-dark".into()); + state.active_tab = ActiveTab::Serving; + let mut wizard = crate::ui::serve_wizard::ServeWizardState { + model: "typed-model".into(), + ..Default::default() + }; + wizard.editor = Some(crate::ui::serve_wizard::InlineEditor { + field: crate::ui::serve_wizard::Field::Host, + value: "localhost".into(), + cursor: 9, + selected: false, + }); + state.serve_wizard = Some(wizard); + + assert!( + !state.should_pane_back_out(crossterm::event::KeyCode::Esc), + "Esc must reach the inline editor instead of closing the wizard" + ); + + // Drive Esc through the real dispatch (mirrors event_loop.rs's + // `state.serve_wizard.is_some()` arm) instead of only checking the + // routing predicate above, so a removed `editor_key` Esc arm turns + // this test red. + let key = crossterm::event::KeyEvent::new( + crossterm::event::KeyCode::Esc, + crossterm::event::KeyModifiers::NONE, + ); + let _ = crate::ui::serve_wizard::on_key( + &mut state.serve_wizard, + &mut state.jobs, + &state.model_recipes, + key, + ); + + let wizard = state + .serve_wizard + .as_ref() + .expect("Esc must not close the wizard"); + assert!( + wizard.editor.is_none(), + "Esc must discard the inline editor" + ); + assert_eq!( + wizard.model, "typed-model", + "the in-progress form remains available to the editor handler" + ); + } + /// `active_overlay_at_root`'s per-manager clauses are a hand-maintained /// enumeration of each manager's nested sub-view fields (documented on /// `OnboardingState`, which lists every struct this covers). Nothing stops @@ -1895,10 +1946,12 @@ mod tests { field: _, model: _, engine_idx: _, - device_idx: _, host: _, + port_mode: _, port: _, managed: _, + advanced_expanded: _, + editor: _, browser, picker, approval, diff --git a/crates/rocm-dash-tui/src/ui/serve_wizard.rs b/crates/rocm-dash-tui/src/ui/serve_wizard.rs index ed3a5da46..a2b4e84ea 100644 --- a/crates/rocm-dash-tui/src/ui/serve_wizard.rs +++ b/crates/rocm-dash-tui/src/ui/serve_wizard.rs @@ -5,11 +5,18 @@ //! Serve wizard overlay (Phase 3 Wave 1). //! //! The headline operational screen rebuilt on the Wave-0 primitives: a compact -//! form that builds a `rocm serve … --managed` invocation and runs it **through -//! the approval gate and the job-bridge** — never inline, never with a legacy -//! `std::thread::spawn` + `try_recv`. A served model launched here surfaces in -//! the services manager and the dashboard's live `gen_tps` (the D7 wire is -//! already in place via `rocm serve --managed`). +//! **model-first** form that builds a `rocm serve …` invocation and runs it +//! **through the approval gate and the job-bridge** — never inline, never with +//! a legacy `std::thread::spawn` + `try_recv`. A served model launched here +//! surfaces in the services manager and the dashboard's live `gen_tps`. +//! +//! Progressive disclosure: the default form is Model → `Advanced settings` → +//! Launch. Everything else (engine, device policy, host, port, mode) is +//! automatic and lives behind the inline `Advanced settings` row — expanded in +//! place, never as a second modal. Because the defaults are genuinely +//! automatic, the default invocation is exactly `rocm serve MODEL`: the CLI +//! stays the single resolution authority for engine choice, the GPU-required +//! policy, the loopback host, the port, and managed mode. //! //! The model field can be typed directly (a recipe name, alias, or path) or //! filled from the reusable Wave-0 [`FolderBrowser`] for a local model path @@ -36,28 +43,63 @@ use crate::ui::model_picker::{ModelPicker, ModelRecipeSummary, PickerOutcome, dr use crate::ui::panel::{self, BoxRole}; use crate::ui::theme::Theme; -/// Engine inventory — names mirror `apps/rocm` `engine_inventory()`. Kept -/// TUI-local (a stable, small list) so this layer needs no `rocm-core` dep. -pub const ENGINES: &[&str] = &["lemonade", "vllm", "aim"]; - -/// Device-policy choices. Index 0 omits `--device` entirely (engine default); -/// the rest mirror `rocm-core`'s validated `gpu_required|gpu_preferred|cpu_only`. -pub const DEVICES: &[&str] = &[ - "(engine default)", - "gpu_required", - "gpu_preferred", - "cpu_only", -]; +/// Engine inventory — index 0 is the automatic choice. +/// +/// Automatic emits no `--engine` and lets the CLI resolve one; the rest mirror +/// `apps/rocm` `engine_inventory()`. Kept TUI-local (a stable, small list) so +/// this layer needs no `rocm-core` dep. +pub const ENGINES: &[&str] = &["automatic", "lemonade", "vllm", "aim"]; + +/// Index into [`ENGINES`] that means "let the CLI decide". +pub const ENGINE_AUTO: usize = 0; + +/// The device policy this wizard offers: read-only, GPU-required, automatic. +/// +/// ROCm never falls back to CPU, so there is nothing here to choose — omitting +/// `--device` already resolves to `gpu_required` in the CLI. +pub const DEVICE_POLICY: &str = "GPU required (automatic)"; /// Mirrors `rocm-core::DEFAULT_LOCAL_HOST` / `DEFAULT_LOCAL_PORT` (TUI-local to -/// avoid the dep; the CLI re-applies its own defaults if these are cleared). +/// avoid the dep). The default host is omitted from argv; the default port text +/// is only the starting point for a *Custom* port. const DEFAULT_HOST: &str = "127.0.0.1"; const DEFAULT_PORT: &str = "11435"; -/// The form fields, in vertical order. +/// The shared phrase for an automatic port. +/// +/// The wizard cannot truthfully promise a concrete port before the CLI leases +/// one, so the approval card says exactly this and the launch output prints the +/// resolved endpoint. +pub const AUTO_PORT_NOTE: &str = "Port: automatic; endpoint shown after launch"; + +/// The exact cross-field guidance for a custom host left on an automatic port. +/// +/// Automatic selection is only supported on the canonical loopback host. +pub const CUSTOM_HOST_NEEDS_PORT: &str = "Custom hosts require a custom port; set Port to Custom."; + +/// Loopback hosts the CLI accepts without `--allow-public-bind`. +/// +/// Dash has no public-bind confirmation or endpoint-key UI, so a public bind +/// must be requested deliberately from the CLI instead. +pub const LOOPBACK_HOSTS: &[&str] = &["127.0.0.1", "localhost", "::1"]; + +/// The guidance shown when a public / non-loopback host is typed into Dash. +pub const PUBLIC_HOST_NEEDS_CLI: &str = "Dash serves on loopback only; for a public bind run \ + `rocm serve --host … --port … --allow-public-bind` from the CLI."; + +/// Whether `host` is one of the exact loopback spellings accepted by the CLI. +#[must_use] +pub fn is_loopback_host(host: &str) -> bool { + LOOPBACK_HOSTS.contains(&host) +} + +/// The form fields. Not every field is visible at once — see +/// [`ServeWizardState::visible_fields`]. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Field { Model, + /// Inline progressive-disclosure row (`Advanced settings ▸/▾`). + Advanced, Engine, Device, Host, @@ -66,9 +108,10 @@ pub enum Field { Launch, } -/// Field order; `state.field` indexes this. +/// Full (expanded) field order. pub const FIELDS: &[Field] = &[ Field::Model, + Field::Advanced, Field::Engine, Field::Device, Field::Host, @@ -77,13 +120,95 @@ pub const FIELDS: &[Field] = &[ Field::Launch, ]; +/// Collapsed (default) field order — model-first, then disclosure, then launch. +pub const BASIC_FIELDS: &[Field] = &[Field::Model, Field::Advanced, Field::Launch]; + +/// Port policy: automatic (the CLI leases a free local port) or an explicit +/// custom port typed by the user. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum PortMode { + #[default] + Auto, + Custom, +} + +/// Inline single-line text editor state for the Host / Custom-port rows. +/// +/// On entry the existing value is *selected*: the first printable character +/// replaces it wholesale (the common "retype it" case) while Left/Right, +/// Backspace, and further typing fall back to ordinary cursor editing. +#[derive(Debug, Clone)] +pub struct InlineEditor { + pub field: Field, + pub value: String, + /// Cursor position, in characters, in `0..=value.chars().count()`. + pub cursor: usize, + /// Whether the initial value is still fully selected. + pub selected: bool, +} + +impl InlineEditor { + fn new(field: Field, value: &str) -> Self { + Self { + field, + value: value.to_string(), + cursor: value.chars().count(), + selected: true, + } + } + + fn len(&self) -> usize { + self.value.chars().count() + } + + fn byte_at(&self, cursor: usize) -> usize { + self.value + .char_indices() + .nth(cursor) + .map_or(self.value.len(), |(b, _)| b) + } + + fn insert(&mut self, c: char) { + if self.selected { + self.value.clear(); + self.cursor = 0; + self.selected = false; + } + let at = self.byte_at(self.cursor); + self.value.insert(at, c); + self.cursor += 1; + } + + fn backspace(&mut self) { + if self.selected { + self.value.clear(); + self.cursor = 0; + self.selected = false; + return; + } + if self.cursor == 0 { + return; + } + let from = self.byte_at(self.cursor - 1); + let to = self.byte_at(self.cursor); + self.value.replace_range(from..to, ""); + self.cursor -= 1; + } + + fn move_cursor(&mut self, delta: isize) { + self.selected = false; + let max = self.len().cast_signed(); + self.cursor = (self.cursor.cast_signed() + delta).clamp(0, max) as usize; + } +} + /// An approved-but-not-yet-launched serve invocation. #[derive(Debug, Clone)] pub struct PendingServe { /// Resolved `rocm` binary path (captured at approval time so a later /// `current_exe()` failure can't silently drop an approved launch). pub cmd: String, - /// The argv after the binary (`["serve", model, "--engine", …]`). + /// The argv after the binary (`["serve", model, …]`). pub args: Vec, pub request: ApprovalRequest, pub choice: ApprovalChoice, @@ -92,13 +217,19 @@ pub struct PendingServe { /// Overlay state. `None` on `AppState` means the wizard is closed. #[derive(Debug, Clone)] pub struct ServeWizardState { + /// Index into [`ServeWizardState::visible_fields`]. pub field: usize, pub model: String, pub engine_idx: usize, - pub device_idx: usize, pub host: String, + pub port_mode: PortMode, + /// The custom port text (only meaningful when `port_mode` is `Custom`). pub port: String, pub managed: bool, + /// Whether the inline `Advanced settings` rows are showing. + pub advanced_expanded: bool, + /// Inline text editor; `Some` while editing Host or the custom port. + pub editor: Option, /// Local-path picker (Wave-0 primitive); `Some` while browsing. pub browser: Option, /// Model-recipe picker sub-step; `Some` while choosing a recipe. @@ -116,11 +247,13 @@ impl Default for ServeWizardState { Self { field: 0, model: String::new(), - engine_idx: 0, - device_idx: 0, + engine_idx: ENGINE_AUTO, host: DEFAULT_HOST.to_string(), + port_mode: PortMode::Auto, port: DEFAULT_PORT.to_string(), managed: true, + advanced_expanded: false, + editor: None, browser: None, picker: None, approval: None, @@ -131,98 +264,187 @@ impl Default for ServeWizardState { } impl ServeWizardState { - fn current_field(&self) -> Field { - FIELDS[self.field.min(FIELDS.len() - 1)] + /// The rows the user can actually see and reach right now. + #[must_use] + pub const fn visible_fields(&self) -> &'static [Field] { + if self.advanced_expanded { + FIELDS + } else { + BASIC_FIELDS + } + } + + /// The focused row (clamped — collapsing can never strand focus). + #[must_use] + pub fn current_field(&self) -> Field { + let vis = self.visible_fields(); + vis[self.field.min(vis.len() - 1)] } fn move_field(&mut self, delta: isize) { - let max = FIELDS.len().cast_signed() - 1; - self.field = (self.field.cast_signed() + delta).clamp(0, max) as usize; + let vis = self.visible_fields(); + let cur = self.field.min(vis.len() - 1); + let max = vis.len().cast_signed() - 1; + self.field = (cur.cast_signed() + delta).clamp(0, max) as usize; + } + + /// Focus the given field, expanding Advanced when it is hidden. + fn focus(&mut self, field: Field) { + if !self.visible_fields().contains(&field) { + self.advanced_expanded = true; + } + if let Some(idx) = self.visible_fields().iter().position(|f| *f == field) { + self.field = idx; + } } fn cycle(&mut self, delta: isize) { - match self.current_field() { + let touched = self.current_field(); + match touched { + // Advanced is a choice row too: ← collapses, → expands. Only + // reachable while Advanced owns focus, and Advanced sits at the + // same index in both orders, so focus is preserved either way. + Field::Advanced => self.advanced_expanded = delta > 0, Field::Engine => self.engine_idx = cycle_idx(self.engine_idx, ENGINES.len(), delta), - Field::Device => self.device_idx = cycle_idx(self.device_idx, DEVICES.len(), delta), + Field::Port => { + self.port_mode = match self.port_mode { + PortMode::Auto => PortMode::Custom, + PortMode::Custom => PortMode::Auto, + }; + } Field::Mode => self.managed = !self.managed, _ => {} } + // Cycling Port back to Custom can fix the exact complaint an outstanding + // message is naming (e.g. "set Port to Custom"); without this, that + // message would sit on screen after the toggle that resolved it. + restate_or_clear(self, touched); } + /// Direct typing only applies to the Model row; Host and the custom port + /// are edited through the explicit inline editor. fn type_char(&mut self, c: char) { - match self.current_field() { - Field::Model => self.model.push(c), - Field::Host => self.host.push(c), - // Port accepts digits only — never builds an unparseable `--port`. - Field::Port if c.is_ascii_digit() => self.port.push(c), - _ => {} + if self.current_field() == Field::Model { + self.model.push(c); } } fn backspace(&mut self) { - match self.current_field() { - Field::Model => { - self.model.pop(); - } - Field::Host => { - self.host.pop(); - } - Field::Port => { - self.port.pop(); - } - _ => {} + if self.current_field() == Field::Model { + self.model.pop(); } } - /// Build the `rocm` argv for the current form, or an error message. - fn build_args(&self) -> Result, String> { - // Trimmed, and deliberately so: `self.model` is typed as often as it is - // browser-filled, and three other places already derive from the - // trimmed form — the approval card's title, the job id, and the - // "already running" guard. Staging an untrimmed value here would let - // two launches differing only by trailing whitespace collide on one job - // id while actually running different commands. `host` and `port` below - // follow the same rule. - let model = self.model.trim(); - if model.is_empty() { - return Err("model is required".to_string()); + /// Whether any advanced setting deviates from the automatic defaults. + #[must_use] + pub fn is_customized(&self) -> bool { + self.engine_idx != ENGINE_AUTO + || self.host.trim() != DEFAULT_HOST + || self.port_mode != PortMode::Auto + || !self.managed + } + + /// The `Advanced settings` summary text — the one place a *collapsed* form + /// still tells the truth about overrides, including a broken hidden one. + #[must_use] + pub fn advanced_summary(&self) -> &'static str { + if !self.is_customized() { + return "Automatic"; + } + match self.validate_advanced() { + Err((Field::Port, _)) => "Customized · port needs attention", + Err((Field::Host, _)) => "Customized · host needs attention", + _ => "Customized", + } + } + + /// Validate the whole form. On success returns the concrete port to emit + /// (`None` for automatic); on failure the offending field plus a plain fix. + fn validate(&self) -> Result, (Field, String)> { + if self.model.trim().is_empty() { + return Err((Field::Model, "model is required".to_string())); + } + self.validate_advanced() + } + + /// The host/port half of [`Self::validate`], independent of the model, so + /// the collapsed summary can report a broken hidden override on its own. + fn validate_advanced(&self) -> Result, (Field, String)> { + let host = self.host.trim(); + if host.is_empty() { + return Err(( + Field::Host, + format!("host is required; type an address or restore {DEFAULT_HOST}"), + )); + } + // Dash has no `--allow-public-bind` confirmation and no endpoint-key + // UI, so it must never stage a public bind — even with an explicit + // port. That deliberate step belongs to the CLI. Checked before the + // auto-port rule below: a public host is refused outright regardless + // of port mode, so its error must name that, not dead-end the user + // into switching to a custom port that would be refused anyway. + if !is_loopback_host(host) { + return Err((Field::Host, PUBLIC_HOST_NEEDS_CLI.to_string())); + } + // Automatic port selection is only supported on the canonical loopback + // host — anything else must name its own port. Never invent one. + if host != DEFAULT_HOST && self.port_mode == PortMode::Auto { + return Err((Field::Port, CUSTOM_HOST_NEEDS_PORT.to_string())); } + if self.port_mode == PortMode::Custom { + let Some(port) = parse_port(&self.port) else { + let shown = self.port.trim(); + return Err(( + Field::Port, + format!("port `{shown}` is not a valid 1-65535 value"), + )); + }; + return Ok(Some(port)); + } + Ok(None) + } + + /// Build the `rocm` argv for the current form, or the offending field and + /// an error message. Only explicit overrides are emitted: the automatic + /// form is exactly `serve MODEL`. + fn build_args(&self) -> Result, (Field, String)> { + let port = self.validate()?; + let model = self.model.trim(); let mut args = vec!["serve".to_string(), model.to_string()]; - args.push("--engine".to_string()); - args.push(ENGINES[self.engine_idx.min(ENGINES.len() - 1)].to_string()); - // Index 0 = engine default → omit --device. - if self.device_idx > 0 { - args.push("--device".to_string()); - args.push(DEVICES[self.device_idx.min(DEVICES.len() - 1)].to_string()); + // Automatic engine → the CLI resolves it. + if self.engine_idx != ENGINE_AUTO { + args.push("--engine".to_string()); + args.push(ENGINES[self.engine_idx.min(ENGINES.len() - 1)].to_string()); } + // Device policy is automatic and GPU-required; omitting `--device` + // already means `gpu_required` in the CLI. let host = self.host.trim(); - if !host.is_empty() { + if host != DEFAULT_HOST { args.push("--host".to_string()); args.push(host.to_string()); } - let port = self.port.trim(); - if !port.is_empty() { - // u16 accepts 0, but 0 is not a bindable listen port — reject it so - // the error surfaces in the form, not as a downstream bind failure. - match port.parse::() { - Ok(p) if p > 0 => { - args.push("--port".to_string()); - args.push(p.to_string()); - } - _ => return Err(format!("port `{port}` is not a valid 1–65535 value")), - } + if let Some(port) = port { + args.push("--port".to_string()); + args.push(port.to_string()); } - // Managed (default) hands supervision to the daemon → it shows up in the - // services manager + dashboard gen_tps. Foreground runs in this job. - if self.managed { - args.push("--managed".to_string()); - } else { + // Managed (the default) hands supervision to the daemon → it shows up + // in the services manager + dashboard gen_tps, and needs no flag. + if !self.managed { args.push("--foreground".to_string()); } Ok(args) } } +/// Parse a custom port: digits only, 1–65535. `0` parses as `u16` but is not a +/// bindable listen port, so it is rejected here rather than downstream. +fn parse_port(raw: &str) -> Option { + match raw.trim().parse::() { + Ok(p) if p > 0 => Some(p), + _ => None, + } +} + const fn cycle_idx(cur: usize, len: usize, delta: isize) -> usize { if len == 0 { return 0; @@ -249,14 +471,10 @@ pub fn on_key( if let Some(picker) = w.picker.as_mut() { match picker.on_key(key.code, recipes) { PickerOutcome::Chosen(summary) => { + // Only the model is filled. A recipe's preferred engine is NOT + // forced onto the form: the CLI stays the resolution authority + // and the wizard keeps emitting an automatic engine. w.model = summary.id; - // Pre-select the recipe's preferred engine when it is one this - // wizard lists; otherwise leave the engine choice untouched. - if let Some(eng) = summary.preferred_engine - && let Some(idx) = ENGINES.iter().position(|e| *e == eng) - { - w.engine_idx = idx; - } w.picker = None; } PickerOutcome::Cancelled => w.picker = None, @@ -278,7 +496,7 @@ pub fn on_key( return Vec::new(); } - // 2) Approval modal has focus. + // 3) Approval modal has focus. if let Some(pending) = w.approval.as_mut() { let (choice, verdict) = approval_key(key.code, pending.choice); pending.choice = choice; @@ -294,7 +512,7 @@ pub fn on_key( return Vec::new(); } - // 3) A launch job is showing in the console. + // 4) A launch job is showing in the console. if let Some(job_id) = w.active_job.clone() { match on_console_key(&job_id, jobs, key) { ConsoleOutcome::Cancelled(fx) => return fx, @@ -305,7 +523,15 @@ pub fn on_key( return Vec::new(); } - // 4) Form editing. + // 5) Inline text editor (Host / custom Port) owns the keyboard. It is not a + // modal: the form stays painted underneath, and form navigation keys + // cannot reach hidden rows from here. + if w.editor.is_some() { + editor_key(w, key.code); + return Vec::new(); + } + + // 6) Form editing. match key.code { KeyCode::Esc => *wizard = None, KeyCode::Up => w.move_field(-1), @@ -313,15 +539,24 @@ pub fn on_key( KeyCode::Left => w.cycle(-1), KeyCode::Right => w.cycle(1), KeyCode::Char(' ') if w.current_field() == Field::Mode => w.cycle(1), + KeyCode::Char(' ') if w.current_field() == Field::Advanced => { + w.advanced_expanded = !w.advanced_expanded; + } // Tab on the Model field opens the local-path picker (Wave-0 primitive). KeyCode::Tab if w.current_field() == Field::Model => { let start = std::env::current_dir().unwrap_or_else(|_| std::path::PathBuf::from("/")); w.browser = Some(FolderBrowser::new("Pick a local model path", start)); } - KeyCode::Enter => { - if w.current_field() == Field::Launch { - request_launch(w); - } else if w.current_field() == Field::Model && !recipes.is_empty() { + KeyCode::Enter => match w.current_field() { + Field::Launch => request_launch(w), + // Expand/collapse only ever happens while Advanced owns focus, so + // focus lands back on Advanced in both directions. + Field::Advanced => w.advanced_expanded = !w.advanced_expanded, + Field::Host => w.editor = Some(InlineEditor::new(Field::Host, &w.host)), + Field::Port if w.port_mode == PortMode::Custom => { + w.editor = Some(InlineEditor::new(Field::Port, &w.port)); + } + Field::Model if !recipes.is_empty() => { // On the Model field, Enter opens the recipe picker (the // model_picker sub-step); free-text typing + Tab-browse remain. // Seed the filter with anything already typed so the picker @@ -330,10 +565,9 @@ pub fn on_key( query: w.model.trim().to_string(), selected: 0, }); - } else { - w.move_field(1); } - } + _ => w.move_field(1), + }, KeyCode::Backspace => w.backspace(), KeyCode::Char(c) => w.type_char(c), _ => {} @@ -341,25 +575,128 @@ pub fn on_key( Vec::new() } +/// Drive the inline editor. Enter accepts, Escape discards, and nothing here +/// can open another overlay. +fn editor_key(w: &mut ServeWizardState, code: KeyCode) { + if w.editor.is_none() { + return; + } + match code { + KeyCode::Esc => { + // The form's host/port are only written on Enter, so discarding + // the editor here never mutates them. + w.editor = None; + // Keep the complaint only while it is still true; a different one + // (about a field this Esc never touched) is cleared, not surfaced + // or focused. `request_launch` still reports it on launch. + if let Some(current) = w.message.take() + && w.validate().is_err_and(|(_, msg)| msg == current) + { + w.message = Some(current); + } + } + KeyCode::Left | KeyCode::Right => { + let delta = if code == KeyCode::Left { -1 } else { 1 }; + if let Some(ed) = w.editor.as_mut() { + ed.move_cursor(delta); + } + } + KeyCode::Backspace => { + if let Some(ed) = w.editor.as_mut() { + ed.backspace(); + } + } + KeyCode::Enter => { + let Some((field, value)) = w.editor.as_ref().map(|e| (e.field, e.value.clone())) else { + return; + }; + match field { + Field::Host => { + let trimmed = value.trim(); + if trimmed.is_empty() { + // Keep the editor open with a plain fix rather than + // silently accepting an unusable host. Esc discards + // the edit and leaves `w.host` as it was before this + // edit started, which is not always `DEFAULT_HOST` — + // the user may be re-editing an already-custom host. + // Name the value Esc actually restores, not a + // hardcoded default that may not match it. + let current_host = w.host.clone(); + w.message = Some(format!( + "host cannot be empty; type an address or press Esc to restore {current_host}" + )); + return; + } + w.host = trimmed.to_string(); + } + Field::Port => { + if value.trim().is_empty() { + // Mirror the Host rule: an empty custom port is not a + // value, so keep the editor open with a plain fix. + w.message = Some( + "port cannot be empty; type a 1-65535 value or press Esc to cancel" + .to_string(), + ); + return; + } + // Digits-only is enforced on input; range validation happens + // at review time so an in-progress value stays visible. + w.port = value; + } + _ => {} + } + restate_or_clear(w, field); + w.editor = None; + } + KeyCode::Char(c) => { + if let Some(ed) = w.editor.as_mut() { + if ed.field == Field::Port && !c.is_ascii_digit() { + return; + } + ed.insert(c); + } + } + _ => {} + } +} + +/// An actual edit to `touched` (a committed Enter, or a Port-mode cycle) can +/// only confirm or deny a complaint about `touched` itself. A complaint about +/// some other, untouched field would be a surprise the user never triggered +/// (e.g. fixing a blank host surfacing "model is required"), so clear it +/// silently instead; `request_launch` still surfaces it if they try to launch. +fn restate_or_clear(w: &mut ServeWizardState, touched: Field) { + if w.message.is_some() { + w.message = match w.validate() { + Ok(_) => None, + Err((field, msg)) if field == touched => Some(msg), + Err(_) => None, + }; + } +} + /// Validate the form and stage an approval (no job runs until approved). fn request_launch(w: &mut ServeWizardState) { match w.build_args() { Ok(args) => { let cmd = resolve_exe(); let cmdline = format!("{} {}", exe_label(&cmd), display_args(&args)); - let request = ApprovalRequest::new( - format!("serve “{}”", w.model.trim()), - vec![ - cmdline, - String::new(), - "This launches a local model server through the ROCm CLI.".to_string(), - if w.managed { - "Managed: it will appear in the services manager and dashboard.".to_string() - } else { - "Foreground: it runs in this job console until stopped.".to_string() - }, - ], - ); + let mut body = vec![ + cmdline, + String::new(), + "This launches a local model server through the ROCm CLI.".to_string(), + ]; + // Only claim an automatic port once the combination is valid — the + // concrete endpoint is printed by the CLI after it leases one. + if w.port_mode == PortMode::Auto { + body.push(AUTO_PORT_NOTE.to_string()); + } + body.push(if w.managed { + "Managed: it will appear in the services manager and dashboard.".to_string() + } else { + "Foreground: it runs in this job console until stopped.".to_string() + }); + let request = ApprovalRequest::new(format!("serve “{}”", w.model.trim()), body); w.message = None; w.approval = Some(PendingServe { cmd, @@ -368,7 +705,12 @@ fn request_launch(w: &mut ServeWizardState) { choice: ApprovalChoice::default(), }); } - Err(msg) => w.message = Some(msg), + Err((field, msg)) => { + // Move focus to the problem — expanding Advanced when the offending + // row is hidden — so the fix is always visible, never guessed at. + w.focus(field); + w.message = Some(msg); + } } } @@ -435,10 +777,12 @@ pub fn draw_serve_wizard( .split(inner); let has_recipes = !recipes.is_empty(); - let lines: Vec = FIELDS + let vis = w.visible_fields(); + let focused = w.field.min(vis.len() - 1); + let lines: Vec = vis .iter() .enumerate() - .map(|(i, field)| field_line(*field, i == w.field, w, has_recipes, theme)) + .map(|(i, field)| field_line(*field, i == focused, w, has_recipes, theme)) .collect(); f.render_widget(Paragraph::new(lines), rows[0]); @@ -451,14 +795,9 @@ pub fn draw_serve_wizard( rows[1], ); - let hint = if recipes.is_empty() { - "↑↓ field · ←→ change · Tab browse (model) · Enter next/launch · Esc close" - } else { - "↑↓ field · ←→ change · Enter pick (model)/next/launch · Tab browse · Esc close" - }; f.render_widget( Paragraph::new(Line::from(Span::styled( - hint, + footer_hint(w, has_recipes), Style::default().fg(theme.muted), ))), rows[2], @@ -476,6 +815,41 @@ pub fn draw_serve_wizard( } } +/// Footer help, derived from focus and edit state. Every hint names the row's +/// role in words (`choice`, `editable`, `read only`) so meaning never depends +/// on colour or glyphs alone. +fn footer_hint(w: &ServeWizardState, has_recipes: bool) -> &'static str { + if let Some(ed) = &w.editor { + return match ed.field { + Field::Port => { + "editing port · ←→ cursor · Backspace delete · Enter accept · Esc cancel" + } + _ => "editing host · ←→ cursor · Backspace delete · Enter accept · Esc cancel", + }; + } + match w.current_field() { + Field::Model if has_recipes => { + "editable · Enter pick a recipe · type a name · Tab browse a path · Esc close" + } + Field::Model => "editable · type a name or path · Tab browse a path · Esc close", + Field::Advanced if w.advanced_expanded => { + "Enter or ← collapse advanced settings · defaults stay automatic" + } + Field::Advanced => "Enter or → expand advanced settings · defaults stay automatic", + Field::Engine => "choice · ←→ pick an engine · automatic lets the CLI decide", + Field::Device => "read only · GPU required · ROCm never falls back to CPU", + Field::Host => "editable · Enter to edit the host · loopback only in Dash", + Field::Port if w.port_mode == PortMode::Custom => { + "choice · ←→ back to automatic · Enter to edit the port" + } + Field::Port => "choice · ←→ switch to custom · automatic picks a free local port", + Field::Mode => "choice · ←→ managed or foreground", + Field::Launch => "Enter to review the exact command · nothing runs before you approve", + } +} + +/// Render one row. Choice values carry chevrons, editable values brackets, and +/// the automatic device policy is plain read-only text. fn field_line<'a>( field: Field, selected: bool, @@ -483,43 +857,6 @@ fn field_line<'a>( has_recipes: bool, theme: &Theme, ) -> Line<'a> { - let model_placeholder = if has_recipes { - "(Enter to pick a recipe · type a name · Tab to browse)" - } else { - "(type a name / path, or Tab to browse)" - }; - let (label, value): (&str, String) = match field { - Field::Model => ( - "Model", - crate::ui::format::display_or_placeholder(&w.model, model_placeholder), - ), - Field::Engine => ( - "Engine", - ENGINES[w.engine_idx.min(ENGINES.len() - 1)].to_string(), - ), - Field::Device => ( - "Device", - DEVICES[w.device_idx.min(DEVICES.len() - 1)].to_string(), - ), - Field::Host => ( - "Host", - crate::ui::format::display_or_placeholder(&w.host, "(engine default)"), - ), - Field::Port => ( - "Port", - crate::ui::format::display_or_placeholder(&w.port, "(engine default)"), - ), - Field::Mode => ( - "Mode", - if w.managed { - "managed".to_string() - } else { - "foreground".to_string() - }, - ), - Field::Launch => ("", String::new()), - }; - if field == Field::Launch { let style = if selected { Style::default() @@ -545,13 +882,126 @@ fn field_line<'a>( } else { Style::default().fg(theme.fg) }; - Line::from(vec![ + + if field == Field::Advanced { + let chevron = if w.advanced_expanded { "▾" } else { "▸" }; + return Line::from(vec![ + Span::styled(marker, label_style), + Span::styled("Advanced settings ", label_style), + Span::styled(chevron, label_style), + Span::styled(format!(" {}", w.advanced_summary()), value_style), + ]); + } + + let model_placeholder = if has_recipes { + "(Enter to pick a recipe · type a name · Tab to browse)" + } else { + "(type a name / path, or Tab to browse)" + }; + + let label = match field { + Field::Model => "Model", + Field::Engine => "Engine", + Field::Device => "Device", + Field::Host => "Host", + Field::Port => "Port", + Field::Mode => "Mode", + Field::Advanced | Field::Launch => unreachable!(), + }; + + let mut spans = vec![ Span::styled(marker, label_style), Span::styled(format!("{label:<8}"), label_style), - Span::styled(value, value_style), - ]) + ]; + + // The editor, when open, replaces its own row's value with a live caret. + let editing = w.editor.as_ref().filter(|e| e.field == field); + + match field { + Field::Model => spans.push(Span::styled( + bracketed(&w.model, model_placeholder), + value_style, + )), + Field::Engine => spans.push(Span::styled( + chevroned(ENGINES[w.engine_idx.min(ENGINES.len() - 1)]), + value_style, + )), + // Read-only: no chevrons, no brackets — nothing to change here. + Field::Device => spans.push(Span::styled(DEVICE_POLICY, value_style)), + Field::Host => { + if let Some(ed) = editing { + spans.extend(editor_spans(ed, value_style, theme)); + } else { + spans.push(Span::styled(bracketed(&w.host, "(unset)"), value_style)); + } + } + Field::Port => { + let mode = match w.port_mode { + PortMode::Auto => "automatic", + PortMode::Custom => "custom", + }; + spans.push(Span::styled(chevroned(mode), value_style)); + if w.port_mode == PortMode::Custom { + spans.push(Span::styled(" ", value_style)); + if let Some(ed) = editing { + spans.extend(editor_spans(ed, value_style, theme)); + } else { + spans.push(Span::styled(bracketed(&w.port, "(unset)"), value_style)); + } + } + } + Field::Mode => spans.push(Span::styled( + chevroned(if w.managed { "managed" } else { "foreground" }), + value_style, + )), + Field::Advanced | Field::Launch => unreachable!(), + } + + Line::from(spans) +} + +/// Editable value with a visible caret (and a reversed run while the initial +/// value is still selected), rendered so the text stays contiguous. +fn editor_spans<'a>(ed: &'a InlineEditor, base: Style, theme: &Theme) -> Vec> { + let caret = Style::default() + .bg(theme.accent) + .fg(theme.bg) + .add_modifier(Modifier::BOLD); + let mut spans = vec![Span::styled("[", base)]; + if ed.selected && !ed.value.is_empty() { + spans.push(Span::styled(ed.value.as_str(), caret)); + } else { + let chars: Vec = ed.value.chars().collect(); + let cut = ed.cursor.min(chars.len()); + let before: String = chars[..cut].iter().collect(); + if !before.is_empty() { + spans.push(Span::styled(before, base)); + } + if cut < chars.len() { + spans.push(Span::styled(chars[cut].to_string(), caret)); + let after: String = chars[cut + 1..].iter().collect(); + if !after.is_empty() { + spans.push(Span::styled(after, base)); + } + } else { + spans.push(Span::styled(" ", caret)); + } + } + spans.push(Span::styled("]", base)); + spans +} + +fn bracketed(v: &str, placeholder: &'static str) -> String { + if v.is_empty() { + placeholder.to_string() + } else { + format!("[{v}]") + } } +fn chevroned(v: &str) -> String { + format!("‹ {v} ›") +} #[cfg(test)] mod tests { use super::*; @@ -562,6 +1012,17 @@ mod tests { s.chars().map(|c| key(KeyCode::Char(c))).collect() } + fn feed(wiz: &mut Option, jobs: &mut State, keys: &[KeyCode]) { + for c in keys { + on_key(wiz, jobs, &[], key(*c)); + } + } + + /// Put focus on a row, expanding Advanced when that row is hidden. + fn focus_field(w: &mut ServeWizardState, field: Field) { + w.focus(field); + } + #[test] fn cycle_idx_wraps_both_directions() { assert_eq!(cycle_idx(0, 3, -1), 2); @@ -569,57 +1030,78 @@ mod tests { assert_eq!(cycle_idx(0, 0, 1), 0); } + // ---------------------------------------------------------------- defaults + #[test] - fn default_form_targets_managed_lemonade() { + fn default_form_is_fully_automatic_and_collapsed() { let w = ServeWizardState::default(); - assert!(w.managed); - assert_eq!(ENGINES[w.engine_idx], "lemonade"); - assert_eq!(w.device_idx, 0); // engine default → no --device + assert!(!w.advanced_expanded); + assert_eq!(w.visible_fields(), BASIC_FIELDS); + assert_eq!(w.engine_idx, ENGINE_AUTO); + assert_eq!(ENGINES[w.engine_idx], "automatic"); assert_eq!(w.host, "127.0.0.1"); - assert_eq!(w.port, "11435"); + assert_eq!(w.port_mode, PortMode::Auto); + assert!(w.managed); + assert_eq!(w.advanced_summary(), "Automatic"); + assert!(!w.is_customized()); } + // ------------------------------------------------------------------- argv + #[test] fn build_args_requires_a_model() { let w = ServeWizardState::default(); - assert_eq!(w.build_args().unwrap_err(), "model is required"); + let (field, msg) = w.build_args().unwrap_err(); + assert_eq!(field, Field::Model); + assert_eq!(msg, "model is required"); + } + + #[test] + fn default_argv_is_bare_serve_model() { + let w = ServeWizardState { + model: "qwen".into(), + ..Default::default() + }; + assert_eq!(w.build_args().unwrap(), vec!["serve", "qwen"]); } #[test] - fn build_args_emits_managed_serve_with_defaults() { + fn automatic_defaults_omit_engine_device_host_port_and_managed() { let w = ServeWizardState { model: "qwen".into(), ..Default::default() }; let args = w.build_args().unwrap(); - assert_eq!( - args, - vec![ - "serve", - "qwen", - "--engine", - "lemonade", - "--host", - "127.0.0.1", - "--port", - "11435", - "--managed", - ] - ); + for flag in ["--engine", "--device", "--host", "--port", "--managed"] { + assert!(!args.contains(&flag.to_string()), "{flag} must be omitted"); + } } #[test] - fn build_args_includes_device_when_not_default_and_foreground() { + fn explicit_overrides_emit_only_their_own_flags() { let w = ServeWizardState { model: "glm".into(), - device_idx: 1, // gpu_required + engine_idx: 2, // vllm + host: "localhost".into(), + port_mode: PortMode::Custom, + port: "8000".into(), managed: false, ..Default::default() }; - let args = w.build_args().unwrap(); - assert!(args.windows(2).any(|p| p == ["--device", "gpu_required"])); - assert!(args.contains(&"--foreground".to_string())); - assert!(!args.contains(&"--managed".to_string())); + assert_eq!( + w.build_args().unwrap(), + vec![ + "serve", + "glm", + "--engine", + "vllm", + "--host", + "localhost", + "--port", + "8000", + "--foreground", + ] + ); } #[test] @@ -638,96 +1120,805 @@ mod tests { } #[test] - fn build_args_rejects_bad_port() { + fn custom_port_on_default_host_emits_port_only() { let w = ServeWizardState { model: "m".into(), - port: "99999".into(), + port_mode: PortMode::Custom, + port: "11500".into(), ..Default::default() }; - assert!(w.build_args().unwrap_err().contains("port")); + assert_eq!( + w.build_args().unwrap(), + vec!["serve", "m", "--port", "11500"] + ); + } + + #[test] + fn build_args_rejects_out_of_range_and_zero_ports() { + for bad in ["99999", "0", "", " "] { + let w = ServeWizardState { + model: "m".into(), + port_mode: PortMode::Custom, + port: bad.into(), + ..Default::default() + }; + let (field, msg) = w.build_args().unwrap_err(); + assert_eq!(field, Field::Port, "{bad:?}"); + assert!(msg.contains("1-65535"), "{bad:?}: {msg}"); + } } #[test] - fn build_args_rejects_port_zero() { - // u16 parses 0, but it is not a bindable listen port — must be rejected - // in the form, matching the "1–65535" message. + fn empty_host_is_rejected_with_a_plain_fix() { let w = ServeWizardState { model: "m".into(), - port: "0".into(), + host: " ".into(), ..Default::default() }; - assert!(w.build_args().unwrap_err().contains("port")); + let (field, msg) = w.build_args().unwrap_err(); + assert_eq!(field, Field::Host); + assert!(msg.contains("host is required"), "{msg}"); } #[test] - fn port_field_accepts_digits_only() { - let mut w = ServeWizardState::default(); - w.port.clear(); - w.field = FIELDS.iter().position(|f| *f == Field::Port).unwrap(); - for k in typed("80a0") { - // route through type_char via the field - if let KeyCode::Char(c) = k.code { - w.type_char(c); - } + fn noncanonical_host_with_auto_port_is_blocked_with_the_contract_message() { + // Loopback but not the canonical default: auto-port is refused with the + // "set Port to Custom" guidance, which actually resolves the problem. + for host in ["localhost", "::1"] { + let w = ServeWizardState { + model: "m".into(), + host: host.into(), + port_mode: PortMode::Auto, + ..Default::default() + }; + let (field, msg) = w.build_args().unwrap_err(); + assert_eq!(field, Field::Port, "{host}"); + assert_eq!(msg, CUSTOM_HOST_NEEDS_PORT, "{host}"); + } + // Not loopback at all: refused outright regardless of port mode, so the + // error must point at the real blocker instead of dead-ending the user + // into a custom port that would be refused anyway. + for host in ["0.0.0.0", "192.168.1.5"] { + let w = ServeWizardState { + model: "m".into(), + host: host.into(), + port_mode: PortMode::Auto, + ..Default::default() + }; + let (field, msg) = w.build_args().unwrap_err(); + assert_eq!(field, Field::Host, "{host}"); + assert_eq!(msg, PUBLIC_HOST_NEEDS_CLI, "{host}"); } - assert_eq!(w.port, "800"); } #[test] - fn launch_requires_approval_then_spawns_job() { - let mut wiz = Some(ServeWizardState::default()); - let mut jobs = State::default(); - // Fill the model by typing on the Model field (field 0 by default). - for k in typed("qwen") { - on_key(&mut wiz, &mut jobs, &[], k); + fn exact_cli_loopback_hosts_with_custom_port_are_allowed() { + for host in ["127.0.0.1", "localhost", "::1"] { + let w = ServeWizardState { + model: "m".into(), + host: host.into(), + port_mode: PortMode::Custom, + port: "9000".into(), + ..Default::default() + }; + let expected = if host == DEFAULT_HOST { + vec!["serve", "m", "--port", "9000"] + } else { + vec!["serve", "m", "--host", host, "--port", "9000"] + }; + assert_eq!(w.build_args().unwrap(), expected, "{host}"); } - assert_eq!(wiz.as_ref().unwrap().model, "qwen"); - // Jump to Launch and press Enter → approval staged, NO job yet. - let launch_idx = FIELDS.iter().position(|f| *f == Field::Launch).unwrap(); - wiz.as_mut().unwrap().field = launch_idx; - let fx = on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Enter)); - assert!(fx.is_empty(), "launch must not run before approval"); - assert!(wiz.as_ref().unwrap().approval.is_some()); - assert!(jobs.jobs.is_empty()); + } - // Approve → exactly one SpawnJob, job registered, console active. - let fx = on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Char('y'))); - assert_eq!(fx.len(), 1); - assert!(matches!(fx[0], SideEffect::SpawnJob { .. })); - let w = wiz.as_ref().unwrap(); - assert!(w.approval.is_none()); - assert_eq!(w.active_job.as_deref(), Some("serve-qwen")); - assert_eq!(jobs.jobs.len(), 1); + #[test] + fn public_host_never_reaches_approval_even_with_a_custom_port() { + // Dash has no --allow-public-bind confirmation and no endpoint-key UI, + // so a public bind must be requested from the CLI instead. + for host in [ + "0.0.0.0", + "192.168.1.5", + "::", + "example.invalid", + "LocalHost", + "[::1]", + ] { + let w = ServeWizardState { + model: "m".into(), + host: host.into(), + port_mode: PortMode::Custom, + port: "8000".into(), + ..Default::default() + }; + let (field, msg) = w.build_args().unwrap_err(); + assert_eq!(field, Field::Host, "{host}"); + assert_eq!(msg, PUBLIC_HOST_NEEDS_CLI, "{host}"); + } } #[test] - fn empty_model_launch_sets_message_not_approval() { - let mut wiz = Some(ServeWizardState::default()); + fn public_host_launch_stages_no_approval() { + let mut wiz = Some(ServeWizardState { + model: "m".into(), + host: "0.0.0.0".into(), + port_mode: PortMode::Custom, + port: "8000".into(), + ..Default::default() + }); let mut jobs = State::default(); - let launch_idx = FIELDS.iter().position(|f| *f == Field::Launch).unwrap(); - wiz.as_mut().unwrap().field = launch_idx; + wiz.as_mut().unwrap().focus(Field::Launch); let fx = on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Enter)); assert!(fx.is_empty()); let w = wiz.as_ref().unwrap(); - assert!(w.approval.is_none()); - assert_eq!(w.message.as_deref(), Some("model is required")); + assert!(w.approval.is_none(), "a public bind never reaches approval"); + assert!(w.advanced_expanded, "the offending row is revealed"); + assert_eq!(w.current_field(), Field::Host); + assert_eq!(w.message.as_deref(), Some(PUBLIC_HOST_NEEDS_CLI)); + assert!(jobs.jobs.is_empty()); } #[test] - fn deny_cancels_without_spawning() { + fn is_loopback_host_accepts_only_the_documented_spellings() { + for ok in ["127.0.0.1", "localhost", "::1"] { + assert!(is_loopback_host(ok), "{ok}"); + } + for bad in [ + "0.0.0.0", + "127.0.0.2", + "::", + "192.168.1.5", + "", + "LocalHost", + "[::1]", + ] { + assert!(!is_loopback_host(bad), "{bad}"); + } + } + + // -------------------------------------------------------- disclosure/focus + + #[test] + fn advanced_expands_and_collapses_only_while_focused_and_keeps_focus() { let mut wiz = Some(ServeWizardState::default()); let mut jobs = State::default(); - wiz.as_mut().unwrap().model = "m".into(); - wiz.as_mut().unwrap().field = FIELDS.iter().position(|f| *f == Field::Launch).unwrap(); - on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Enter)); - let fx = on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Char('n'))); - assert!(fx.is_empty()); - assert!(wiz.as_ref().unwrap().approval.is_none()); - assert!(jobs.jobs.is_empty()); + // Model has focus: Right must not expand anything. + feed(&mut wiz, &mut jobs, &[KeyCode::Right]); + assert!(!wiz.as_ref().unwrap().advanced_expanded); + + // Down → Advanced, Enter expands, focus stays on Advanced. + feed(&mut wiz, &mut jobs, &[KeyCode::Down, KeyCode::Enter]); + { + let w = wiz.as_ref().unwrap(); + assert!(w.advanced_expanded); + assert_eq!(w.visible_fields(), FIELDS); + assert_eq!(w.current_field(), Field::Advanced); + } + // Enter collapses again, still on Advanced. + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + { + let w = wiz.as_ref().unwrap(); + assert!(!w.advanced_expanded); + assert_eq!(w.current_field(), Field::Advanced); + } + // ← collapses / → expands from the same row. + feed(&mut wiz, &mut jobs, &[KeyCode::Right]); + assert!(wiz.as_ref().unwrap().advanced_expanded); + feed(&mut wiz, &mut jobs, &[KeyCode::Left]); + assert!(!wiz.as_ref().unwrap().advanced_expanded); + assert_eq!(wiz.as_ref().unwrap().current_field(), Field::Advanced); } #[test] - fn esc_closes_when_idle() { + fn space_toggles_advanced_only_while_focused() { + let mut wiz = Some(ServeWizardState::default()); + let mut jobs = State::default(); + // Model has focus: Space must not expand anything. + feed(&mut wiz, &mut jobs, &[KeyCode::Char(' ')]); + assert!(!wiz.as_ref().unwrap().advanced_expanded); + + focus_field(wiz.as_mut().unwrap(), Field::Advanced); + feed(&mut wiz, &mut jobs, &[KeyCode::Char(' ')]); + assert!(wiz.as_ref().unwrap().advanced_expanded); + feed(&mut wiz, &mut jobs, &[KeyCode::Char(' ')]); + assert!(!wiz.as_ref().unwrap().advanced_expanded); + } + + #[test] + fn expanded_focus_order_is_model_advanced_engine_device_host_port_mode_launch() { + let mut w = ServeWizardState { + advanced_expanded: true, + ..Default::default() + }; + let mut seen = Vec::new(); + for _ in 0..FIELDS.len() { + seen.push(w.current_field()); + w.move_field(1); + } + assert_eq!(seen, FIELDS.to_vec()); + // Clamped at the end. + assert_eq!(w.current_field(), Field::Launch); + } + + #[test] + fn collapsing_preserves_overrides_and_reports_customized() { + let mut w = ServeWizardState { + advanced_expanded: true, + engine_idx: 1, + port_mode: PortMode::Custom, + port: "8000".into(), + ..Default::default() + }; + w.advanced_expanded = false; + assert_eq!(w.engine_idx, 1); + assert_eq!(w.port, "8000"); + assert_eq!(w.advanced_summary(), "Customized"); + } + + #[test] + fn invalid_hidden_custom_port_says_it_needs_attention() { + let w = ServeWizardState { + model: "m".into(), + port_mode: PortMode::Custom, + port: "abc".into(), + ..Default::default() + }; + assert!(!w.advanced_expanded); + assert_eq!(w.advanced_summary(), "Customized · port needs attention"); + assert!(w.build_args().is_err(), "launch stays blocked"); + } + + #[test] + fn out_of_range_hidden_port_also_needs_attention() { + let w = ServeWizardState { + port_mode: PortMode::Custom, + port: "70000".into(), + ..Default::default() + }; + assert_eq!(w.advanced_summary(), "Customized · port needs attention"); + } + + #[test] + fn invalid_hidden_host_says_it_needs_attention() { + let w = ServeWizardState { + host: "0.0.0.0".into(), + ..Default::default() + }; + assert!(!w.advanced_expanded); + assert_eq!(w.advanced_summary(), "Customized · host needs attention"); + } + + #[test] + fn invalid_combination_expands_advanced_and_focuses_the_problem() { + let mut wiz = Some(ServeWizardState { + model: "m".into(), + host: "localhost".into(), + ..Default::default() + }); + let mut jobs = State::default(); + wiz.as_mut().unwrap().field = 2; // Launch (collapsed order) + let fx = on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Enter)); + assert!(fx.is_empty(), "nothing may run"); + let w = wiz.as_ref().unwrap(); + assert!(w.approval.is_none(), "invalid combos never reach approval"); + assert!(w.advanced_expanded, "the hidden problem is revealed"); + assert_eq!(w.current_field(), Field::Port); + assert_eq!(w.message.as_deref(), Some(CUSTOM_HOST_NEEDS_PORT)); + assert_eq!(w.host, "localhost", "both values preserved"); + assert_eq!(w.port_mode, PortMode::Auto); + } + + // --------------------------------------------------------------- controls + + #[test] + fn left_right_cycles_engine_including_automatic() { + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Engine); + feed(&mut wiz, &mut jobs, &[KeyCode::Right]); + assert_eq!(ENGINES[wiz.as_ref().unwrap().engine_idx], "lemonade"); + feed(&mut wiz, &mut jobs, &[KeyCode::Right]); + assert_eq!(ENGINES[wiz.as_ref().unwrap().engine_idx], "vllm"); + feed(&mut wiz, &mut jobs, &[KeyCode::Right]); + assert_eq!(ENGINES[wiz.as_ref().unwrap().engine_idx], "aim"); + feed(&mut wiz, &mut jobs, &[KeyCode::Right]); + assert_eq!(wiz.as_ref().unwrap().engine_idx, ENGINE_AUTO, "wraps"); + } + + #[test] + fn left_right_toggles_port_mode_only_when_focused() { + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Device); + feed(&mut wiz, &mut jobs, &[KeyCode::Right]); + assert_eq!( + wiz.as_ref().unwrap().port_mode, + PortMode::Auto, + "Device is read-only and cannot change the port" + ); + focus_field(wiz.as_mut().unwrap(), Field::Port); + feed(&mut wiz, &mut jobs, &[KeyCode::Right]); + assert_eq!(wiz.as_ref().unwrap().port_mode, PortMode::Custom); + feed(&mut wiz, &mut jobs, &[KeyCode::Left]); + assert_eq!(wiz.as_ref().unwrap().port_mode, PortMode::Auto); + } + + #[test] + fn cycling_port_to_custom_clears_the_outstanding_complaint() { + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + model: "m".into(), + host: "localhost".into(), + ..Default::default() + }); + wiz.as_mut().unwrap().message = Some(CUSTOM_HOST_NEEDS_PORT.to_string()); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Port); + feed(&mut wiz, &mut jobs, &[KeyCode::Right]); + let w = wiz.as_ref().unwrap(); + assert_eq!(w.port_mode, PortMode::Custom); + assert!(w.message.is_none(), "the fix it named just happened"); + } + + #[test] + fn mode_toggles_between_managed_and_foreground() { + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Mode); + feed(&mut wiz, &mut jobs, &[KeyCode::Char(' ')]); + assert!(!wiz.as_ref().unwrap().managed); + feed(&mut wiz, &mut jobs, &[KeyCode::Left]); + assert!(wiz.as_ref().unwrap().managed); + } + + #[test] + fn typing_only_edits_the_model_row() { + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Host); + for k in typed("zzz") { + on_key(&mut wiz, &mut jobs, &[], k); + } + assert_eq!( + wiz.as_ref().unwrap().host, + "127.0.0.1", + "host only changes through the inline editor" + ); + focus_field(wiz.as_mut().unwrap(), Field::Model); + for k in typed("qwen") { + on_key(&mut wiz, &mut jobs, &[], k); + } + assert_eq!(wiz.as_ref().unwrap().model, "qwen"); + feed(&mut wiz, &mut jobs, &[KeyCode::Backspace]); + assert_eq!(wiz.as_ref().unwrap().model, "qwe"); + } + + // ----------------------------------------------------------- inline editor + + #[test] + fn host_editor_selects_on_entry_and_first_char_replaces() { + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Host); + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + { + let ed = wiz.as_ref().unwrap().editor.as_ref().unwrap(); + assert_eq!(ed.field, Field::Host); + assert!(ed.selected); + assert_eq!(ed.value, "127.0.0.1"); + } + for k in typed("0.0.0.0") { + on_key(&mut wiz, &mut jobs, &[], k); + } + assert_eq!( + wiz.as_ref().unwrap().editor.as_ref().unwrap().value, + "0.0.0.0" + ); + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + let w = wiz.as_ref().unwrap(); + assert!(w.editor.is_none()); + assert_eq!(w.host, "0.0.0.0"); + } + + #[test] + fn editor_cursor_insert_and_backspace() { + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + host: "abc".into(), + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Host); + feed(&mut wiz, &mut jobs, &[KeyCode::Enter, KeyCode::Left]); + // Left cleared the selection and moved the cursor to before 'c'. + { + let ed = wiz.as_ref().unwrap().editor.as_ref().unwrap(); + assert!(!ed.selected); + assert_eq!(ed.cursor, 2); + } + on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Char('X'))); + assert_eq!(wiz.as_ref().unwrap().editor.as_ref().unwrap().value, "abXc"); + feed(&mut wiz, &mut jobs, &[KeyCode::Backspace]); + assert_eq!(wiz.as_ref().unwrap().editor.as_ref().unwrap().value, "abc"); + feed(&mut wiz, &mut jobs, &[KeyCode::Right, KeyCode::Right]); + assert_eq!(wiz.as_ref().unwrap().editor.as_ref().unwrap().cursor, 3); + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + assert_eq!(wiz.as_ref().unwrap().host, "abc"); + } + + #[test] + fn editor_escape_discards_the_edit() { + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Host); + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + for k in typed("nonsense") { + on_key(&mut wiz, &mut jobs, &[], k); + } + feed(&mut wiz, &mut jobs, &[KeyCode::Esc]); + let w = wiz.as_ref().unwrap(); + assert!(w.editor.is_none(), "Esc closes the editor"); + assert!(wiz.is_some(), "Esc in the editor never closes the overlay"); + assert_eq!(w.host, "127.0.0.1", "edit was never committed to the form"); + } + + #[test] + fn host_editor_trims_and_rejects_blank() { + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Host); + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + for k in typed(" ") { + on_key(&mut wiz, &mut jobs, &[], k); + } + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + { + let w = wiz.as_ref().unwrap(); + assert!(w.editor.is_some(), "blank host keeps the editor open"); + assert_eq!(w.host, "127.0.0.1", "form untouched"); + assert!( + w.message + .as_deref() + .unwrap() + .contains("host cannot be empty") + ); + } + for k in typed(" 10.0.0.7 ") { + on_key(&mut wiz, &mut jobs, &[], k); + } + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + let w = wiz.as_ref().unwrap(); + assert!(w.editor.is_none()); + assert_eq!(w.host, "10.0.0.7", "trimmed on accept"); + } + + #[test] + fn blanking_an_already_custom_host_names_that_host_as_the_esc_restore_value() { + // The blank-host message names the value Esc actually restores. If + // the form's host is already a custom value (not DEFAULT_HOST) before + // this edit, the message must name that value, not a hardcoded + // default Esc would not actually restore. + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + host: "10.0.0.7".to_string(), + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Host); + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + for k in typed(" ") { + on_key(&mut wiz, &mut jobs, &[], k); + } + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + let w = wiz.as_ref().unwrap(); + let msg = w.message.as_deref().unwrap(); + assert!(msg.contains("restore 10.0.0.7"), "{msg}"); + assert!(!msg.contains("127.0.0.1"), "{msg}"); + } + + #[test] + fn port_editor_accepts_digits_only() { + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + port_mode: PortMode::Custom, + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Port); + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + for k in typed("80a0!") { + on_key(&mut wiz, &mut jobs, &[], k); + } + assert_eq!(wiz.as_ref().unwrap().editor.as_ref().unwrap().value, "800"); + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + assert_eq!(wiz.as_ref().unwrap().port, "800"); + } + + #[test] + fn port_editor_rejects_an_empty_value_like_host_does() { + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + model: "m".into(), + port_mode: PortMode::Custom, + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Port); + // Enter selects "11435"; Backspace clears the whole selection. + feed(&mut wiz, &mut jobs, &[KeyCode::Enter, KeyCode::Backspace]); + assert_eq!(wiz.as_ref().unwrap().editor.as_ref().unwrap().value, ""); + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + { + let w = wiz.as_ref().unwrap(); + assert!(w.editor.is_some(), "empty port keeps the editor open"); + assert_eq!(w.port, "11435", "form untouched"); + let msg = w.message.as_deref().unwrap(); + assert!(msg.contains("port cannot be empty"), "{msg}"); + assert!(msg.contains("1-65535"), "{msg}"); + } + // Typing a real value accepts normally and clears the complaint. + for k in typed("9001") { + on_key(&mut wiz, &mut jobs, &[], k); + } + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + let w = wiz.as_ref().unwrap(); + assert!(w.editor.is_none()); + assert_eq!(w.port, "9001"); + assert!(w.message.is_none(), "a fixed form stops complaining"); + } + + #[test] + fn escaping_an_editor_recomputes_an_outstanding_complaint() { + // A blank-host complaint must not survive the Esc that restores a + // perfectly good host… + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + model: "m".into(), + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Host); + feed( + &mut wiz, + &mut jobs, + &[KeyCode::Enter, KeyCode::Backspace, KeyCode::Enter], + ); + assert!(wiz.as_ref().unwrap().message.is_some()); + feed(&mut wiz, &mut jobs, &[KeyCode::Esc]); + let w = wiz.as_ref().unwrap(); + assert_eq!(w.host, "127.0.0.1"); + assert!(w.message.is_none(), "stale complaint cleared by the fix"); + + // …but a still-broken form keeps saying why. + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + model: "m".into(), + host: "localhost".into(), + ..Default::default() + }); + wiz.as_mut().unwrap().message = Some(CUSTOM_HOST_NEEDS_PORT.to_string()); + focus_field(wiz.as_mut().unwrap(), Field::Host); + feed(&mut wiz, &mut jobs, &[KeyCode::Enter, KeyCode::Esc]); + let w = wiz.as_ref().unwrap(); + assert_eq!(w.host, "localhost"); + assert_eq!(w.message.as_deref(), Some(CUSTOM_HOST_NEEDS_PORT)); + } + + #[test] + fn escaping_a_host_editor_error_does_not_surface_or_focus_another_fields_complaint() { + // Host edit -> blank Enter -> Esc, with the model still empty: the Host + // complaint is gone after Esc, and "model is required" must not replace + // it or pull focus off Host. + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Host); + feed( + &mut wiz, + &mut jobs, + &[KeyCode::Enter, KeyCode::Backspace, KeyCode::Enter], + ); + assert!(wiz.as_ref().unwrap().message.is_some()); + feed(&mut wiz, &mut jobs, &[KeyCode::Esc]); + let w = wiz.as_ref().unwrap(); + assert!(w.message.is_none(), "{:?}", w.message); + assert_eq!(w.current_field(), Field::Host); + } + + #[test] + fn committing_a_host_edit_that_surfaces_an_unrelated_port_complaint_clears_it_instead() { + // An outstanding complaint about the Host row (blank host) is fixed by + // typing a loopback host the CLI accepts — but that host also needs a + // custom port, a complaint about a field the user never touched in + // this edit. Silently clear it rather than surprise the user with a + // new complaint about Port; `request_launch` still catches it later. + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + model: "m".into(), + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Host); + wiz.as_mut().unwrap().message = + Some("host cannot be empty; type an address or restore 127.0.0.1".to_string()); + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + for k in typed("localhost") { + on_key(&mut wiz, &mut jobs, &[], k); + } + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + let w = wiz.as_ref().unwrap(); + assert_eq!(w.host, "localhost"); + assert!(w.message.is_none(), "not the user's business right now"); + assert_eq!(w.current_field(), Field::Host, "focus stays put"); + } + + #[test] + fn fixing_a_host_complaint_never_surfaces_an_unrelated_model_complaint() { + // Regression test for the exact bug reported: a fresh form with an + // empty Model and an outstanding blank-host complaint. Fixing Host + // must not raise "model is required" — a complaint the user never + // triggered by editing Host. + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Host); + wiz.as_mut().unwrap().message = + Some("host cannot be empty; type an address or restore 127.0.0.1".to_string()); + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + for k in typed("localhost") { + on_key(&mut wiz, &mut jobs, &[], k); + } + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + let w = wiz.as_ref().unwrap(); + assert_eq!(w.host, "localhost"); + assert!(w.message.is_none(), "model's emptiness is not Host's news"); + assert_eq!(w.current_field(), Field::Host, "focus stays put"); + } + + #[test] + fn enter_on_auto_port_advances_instead_of_editing() { + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Port); + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + let w = wiz.as_ref().unwrap(); + assert!(w.editor.is_none()); + assert_eq!(w.current_field(), Field::Mode); + } + + #[test] + fn editor_navigation_keys_cannot_move_form_focus() { + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Host); + let before = wiz.as_ref().unwrap().field; + feed( + &mut wiz, + &mut jobs, + &[KeyCode::Enter, KeyCode::Up, KeyCode::Down], + ); + let w = wiz.as_ref().unwrap(); + assert!(w.editor.is_some()); + assert_eq!(w.field, before, "form navigation is inert while editing"); + } + + // -------------------------------------------------------------- approval + + #[test] + fn launch_requires_approval_then_spawns_job() { + let mut wiz = Some(ServeWizardState::default()); + let mut jobs = State::default(); + for k in typed("qwen") { + on_key(&mut wiz, &mut jobs, &[], k); + } + assert_eq!(wiz.as_ref().unwrap().model, "qwen"); + wiz.as_mut().unwrap().focus(Field::Launch); + let fx = on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Enter)); + assert!(fx.is_empty(), "launch must not run before approval"); + assert!(wiz.as_ref().unwrap().approval.is_some()); + assert!(jobs.jobs.is_empty()); + + let fx = on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Char('y'))); + assert_eq!(fx.len(), 1); + assert!(matches!(fx[0], SideEffect::SpawnJob { .. })); + let w = wiz.as_ref().unwrap(); + assert!(w.approval.is_none()); + assert_eq!(w.active_job.as_deref(), Some("serve-qwen")); + assert_eq!(jobs.jobs.len(), 1); + } + + // This only pins the shared constant into the approval body; the CLI's own + // plan output proves the phrase actually appears before the resolved + // endpoint, in `serve-auto-port-skips-occupied` (serve-31). + #[test] + fn approval_shows_exact_argv_and_the_automatic_port_phrase() { + let mut w = ServeWizardState { + model: "qwen".into(), + ..Default::default() + }; + request_launch(&mut w); + let pending = w.approval.as_ref().unwrap(); + assert_eq!(pending.args, vec!["serve", "qwen"]); + let body = pending.request.body.join("\n"); + assert!(body.contains("serve qwen"), "{body}"); + assert!(body.contains(AUTO_PORT_NOTE), "{body}"); + } + + #[test] + fn approval_omits_the_automatic_phrase_for_a_custom_port() { + let mut w = ServeWizardState { + model: "qwen".into(), + port_mode: PortMode::Custom, + port: "8000".into(), + ..Default::default() + }; + request_launch(&mut w); + let pending = w.approval.as_ref().unwrap(); + assert!(pending.args.windows(2).any(|p| p == ["--port", "8000"])); + assert!(!pending.request.body.join("\n").contains(AUTO_PORT_NOTE)); + } + + #[test] + fn empty_model_launch_sets_message_not_approval() { + let mut wiz = Some(ServeWizardState::default()); + let mut jobs = State::default(); + wiz.as_mut().unwrap().focus(Field::Launch); + let fx = on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Enter)); + assert!(fx.is_empty()); + let w = wiz.as_ref().unwrap(); + assert!(w.approval.is_none()); + assert_eq!(w.message.as_deref(), Some("model is required")); + assert_eq!( + w.current_field(), + Field::Model, + "focus lands on the problem" + ); + } + + #[test] + fn deny_cancels_without_spawning() { + let mut wiz = Some(ServeWizardState::default()); + let mut jobs = State::default(); + wiz.as_mut().unwrap().model = "m".into(); + wiz.as_mut().unwrap().focus(Field::Launch); + on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Enter)); + let fx = on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Char('n'))); + assert!(fx.is_empty()); + assert!(wiz.as_ref().unwrap().approval.is_none()); + assert!(jobs.jobs.is_empty()); + } + + #[test] + fn esc_closes_when_idle() { let mut wiz = Some(ServeWizardState::default()); let mut jobs = State::default(); on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Esc)); @@ -739,7 +1930,7 @@ mod tests { let mut wiz = Some(ServeWizardState::default()); let mut jobs = State::default(); wiz.as_mut().unwrap().model = "m".into(); - wiz.as_mut().unwrap().field = FIELDS.iter().position(|f| *f == Field::Launch).unwrap(); + wiz.as_mut().unwrap().focus(Field::Launch); on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Enter)); on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Char('y'))); assert!(wiz.as_ref().unwrap().active_job.is_some()); @@ -754,7 +1945,7 @@ mod tests { let mut wiz = Some(ServeWizardState::default()); let mut jobs = State::default(); wiz.as_mut().unwrap().model = "m".into(); - wiz.as_mut().unwrap().field = FIELDS.iter().position(|f| *f == Field::Launch).unwrap(); + wiz.as_mut().unwrap().focus(Field::Launch); on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Enter)); on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Char('y'))); assert!(wiz.as_ref().unwrap().active_job.is_some()); @@ -765,7 +1956,7 @@ mod tests { let mut wiz = Some(ServeWizardState::default()); let mut jobs = State::default(); wiz.as_mut().unwrap().model = "m".into(); - wiz.as_mut().unwrap().field = FIELDS.iter().position(|f| *f == Field::Launch).unwrap(); + wiz.as_mut().unwrap().focus(Field::Launch); on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Enter)); on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Char('y'))); let job_id = wiz.as_ref().unwrap().active_job.clone().unwrap(); @@ -787,13 +1978,12 @@ mod tests { // NOT claim success (set active_job) when no SpawnJob was emitted. let mut wiz = Some(ServeWizardState::default()); let mut jobs = State::default(); - let launch = FIELDS.iter().position(|f| *f == Field::Launch).unwrap(); // First launch of "qwen": real spawn. for k in typed("qwen") { on_key(&mut wiz, &mut jobs, &[], k); } - wiz.as_mut().unwrap().field = launch; + wiz.as_mut().unwrap().focus(Field::Launch); on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Enter)); on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Char('y'))); assert_eq!( @@ -807,7 +1997,7 @@ mod tests { for k in typed("qwen") { on_key(&mut wiz2, &mut jobs, &[], k); } - wiz2.as_mut().unwrap().field = launch; + wiz2.as_mut().unwrap().focus(Field::Launch); on_key(&mut wiz2, &mut jobs, &[], key(KeyCode::Enter)); let fx = on_key(&mut wiz2, &mut jobs, &[], key(KeyCode::Char('y'))); // No new SpawnJob, no stale console, an informative message instead. @@ -823,6 +2013,8 @@ mod tests { assert_eq!(jobs.jobs.len(), 1, "still just the one job"); } + // ------------------------------------------------------- picker / browser + #[test] fn tab_on_model_opens_folder_browser() { let mut wiz = Some(ServeWizardState::default()); @@ -836,18 +2028,7 @@ mod tests { } #[test] - fn left_right_cycles_engine() { - let mut wiz = Some(ServeWizardState::default()); - let mut jobs = State::default(); - wiz.as_mut().unwrap().field = FIELDS.iter().position(|f| *f == Field::Engine).unwrap(); - on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Right)); - assert_eq!(wiz.as_ref().unwrap().engine_idx, 1); - on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Left)); - assert_eq!(wiz.as_ref().unwrap().engine_idx, 0); - } - - #[test] - fn enter_on_model_opens_picker_and_choice_fills_model_and_engine() { + fn enter_on_model_opens_picker_and_choice_fills_model_only() { let recipes = vec![ModelRecipeSummary { id: "GLM-4".into(), aliases: vec!["glm".into()], @@ -856,18 +2037,17 @@ mod tests { }]; let mut wiz = Some(ServeWizardState::default()); // field 0 = Model let mut jobs = State::default(); - // Enter on Model opens the picker when recipes exist. on_key(&mut wiz, &mut jobs, &recipes, key(KeyCode::Enter)); assert!(wiz.as_ref().unwrap().picker.is_some()); - // Enter in the picker chooses the (only) recipe → fills model + engine. on_key(&mut wiz, &mut jobs, &recipes, key(KeyCode::Enter)); let w = wiz.as_ref().unwrap(); assert!(w.picker.is_none()); assert_eq!(w.model, "GLM-4"); assert_eq!( - ENGINES[w.engine_idx], "vllm", - "preferred engine pre-selected" + w.engine_idx, ENGINE_AUTO, + "a recipe never forces an engine override — the CLI resolves it" ); + assert_eq!(w.build_args().unwrap(), vec!["serve", "GLM-4"]); } #[test] @@ -876,29 +2056,10 @@ mod tests { let mut jobs = State::default(); on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Enter)); assert!(wiz.as_ref().unwrap().picker.is_none()); - assert_eq!(wiz.as_ref().unwrap().field, 1, "Enter advances to Engine"); - } - - #[test] - fn recipe_with_unknown_preferred_engine_leaves_engine_idx() { - // Pins the silent-fallback contract: a preferred_engine the wizard does - // not list must NOT crash and must leave the engine choice untouched - // (model still filled). Guards future ENGINES vs rocm-core divergence. - let recipes = vec![ModelRecipeSummary { - id: "some-model".into(), - aliases: vec![], - task: "chat".into(), - preferred_engine: Some("not-in-engines-list".into()), - }]; - let mut wiz = Some(ServeWizardState::default()); // engine_idx 0 = lemonade - let mut jobs = State::default(); - on_key(&mut wiz, &mut jobs, &recipes, key(KeyCode::Enter)); // open picker - on_key(&mut wiz, &mut jobs, &recipes, key(KeyCode::Enter)); // choose first - let w = wiz.as_ref().unwrap(); - assert_eq!(w.model, "some-model"); assert_eq!( - w.engine_idx, 0, - "unknown preferred engine leaves the choice" + wiz.as_ref().unwrap().current_field(), + Field::Advanced, + "Enter advances to the disclosure row" ); } @@ -920,7 +2081,6 @@ mod tests { ]; let mut wiz = Some(ServeWizardState::default()); let mut jobs = State::default(); - // Type "qwen" on the Model field, then Enter to open the picker. for k in typed("qwen") { on_key(&mut wiz, &mut jobs, &recipes, k); } @@ -928,7 +2088,6 @@ mod tests { let picker = wiz.as_ref().unwrap().picker.as_ref().unwrap(); assert_eq!(picker.query, "qwen"); assert_eq!(picker.filtered(&recipes).len(), 1, "pre-narrowed to Qwen"); - // Enter chooses the single match. on_key(&mut wiz, &mut jobs, &recipes, key(KeyCode::Enter)); assert_eq!(wiz.as_ref().unwrap().model, "Qwen3-4B"); } @@ -951,6 +2110,8 @@ mod tests { assert!(wiz.is_some(), "picker Esc keeps the wizard open"); } + // ---------------------------------------------------------------- render + fn render(w: &ServeWizardState, jobs: &State) -> String { use ratatui::Terminal; use ratatui::backend::TestBackend; @@ -966,14 +2127,173 @@ mod tests { .collect() } + /// A rendered form row starts with the 2-cell focus marker and an + /// 8-cell-padded label, so `Mode` is `"Mode "` and can never be matched + /// by the `Model` row. Bare substrings are not precise enough here. + fn row_label(label: &str) -> String { + format!("{label:<8}") + } + #[test] - fn snapshot_renders_form_fields() { + fn collapsed_render_shows_only_model_advanced_and_launch() { let w = ServeWizardState::default(); let out = render(&w, &State::default()); assert!(out.contains("Serve a model"), "titled overlay"); - assert!(out.contains("Model"), "model field"); - assert!(out.contains("lemonade"), "default engine shown"); - assert!(out.contains("Launch"), "launch action"); + assert!(out.contains(&row_label("Model")), "model row"); + assert!(out.contains("Advanced settings ▸"), "collapsed disclosure"); + assert!(out.contains("Automatic"), "automatic summary"); + assert!(out.contains("[ Launch ]"), "launch action"); + for hidden in ["Engine", "Device", "Host", "Port", "Mode"] { + assert!( + !out.contains(&row_label(hidden)), + "{hidden} row must stay hidden: {out:?}" + ); + } + } + + #[test] + fn expanded_render_inlines_the_advanced_rows_without_a_second_modal() { + let w = ServeWizardState { + advanced_expanded: true, + ..Default::default() + }; + let out = render(&w, &State::default()); + assert!(out.contains("Advanced settings ▾"), "expanded disclosure"); + for shown in ["Model", "Engine", "Device", "Host", "Port", "Mode"] { + assert!( + out.contains(&row_label(shown)), + "{shown} row missing: {out:?}" + ); + } + assert!(out.contains("[ Launch ]"), "launch action"); + // Role affordances: chevrons for choices, brackets for editable text, + // plain read-only text for the GPU-required policy. + assert!(out.contains("‹ automatic ›"), "choice chevrons: {out:?}"); + assert!(out.contains("[127.0.0.1]"), "editable brackets: {out:?}"); + assert!( + out.contains(DEVICE_POLICY), + "read-only device text: {out:?}" + ); + assert!(!out.contains("Review:"), "no second modal"); + } + + #[test] + fn engine_only_override_summarises_as_customized() { + let w = ServeWizardState { + engine_idx: ENGINE_AUTO + 1, + ..Default::default() + }; + assert!(w.is_customized()); + assert_eq!(w.advanced_summary(), "Customized"); + } + + #[test] + fn foreground_only_form_summarises_as_customized() { + let w = ServeWizardState { + managed: false, + ..Default::default() + }; + assert!(w.is_customized()); + assert_eq!(w.advanced_summary(), "Customized"); + } + + #[test] + fn loopback_name_with_automatic_port_summarises_as_needing_attention() { + let w = ServeWizardState { + model: "m".into(), + host: "localhost".into(), + ..Default::default() + }; + assert!(w.build_args().is_err(), "launch is refused"); + assert_eq!(w.advanced_summary(), "Customized · port needs attention"); + } + + #[test] + fn committing_a_bad_port_restates_the_port_complaint() { + let mut wiz = Some(ServeWizardState { + model: "m".into(), + advanced_expanded: true, + port_mode: PortMode::Custom, + port: "abc".into(), + message: Some("port `abc` is not a valid 1-65535 value".into()), + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Port); + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + for c in typed("99999") { + on_key(&mut wiz, &mut jobs, &[], c); + } + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + let w = wiz.as_ref().unwrap(); + assert_eq!( + w.message.as_deref(), + Some("port `99999` is not a valid 1-65535 value"), + "the complaint is re-stated for the value just committed" + ); + } + + #[test] + fn open_editor_renders_the_typed_text() { + let mut wiz = Some(ServeWizardState { + advanced_expanded: true, + ..Default::default() + }); + let mut jobs = State::default(); + focus_field(wiz.as_mut().unwrap(), Field::Host); + feed(&mut wiz, &mut jobs, &[KeyCode::Enter]); + for c in typed("nope") { + on_key(&mut wiz, &mut jobs, &[], c); + } + let out = render(wiz.as_ref().unwrap(), &jobs); + assert!( + out.contains("nope"), + "typed text must be on screen: {out:?}" + ); + } + + #[test] + fn collapsed_render_reports_a_broken_hidden_port() { + let w = ServeWizardState { + model: "m".into(), + port_mode: PortMode::Custom, + port: "abc".into(), + ..Default::default() + }; + let out = render(&w, &State::default()); + assert!(out.contains("port needs attention"), "{out:?}"); + } + + #[test] + fn footer_names_the_role_of_the_focused_row() { + let mut w = ServeWizardState { + advanced_expanded: true, + ..Default::default() + }; + w.focus(Field::Engine); + assert!(render(&w, &State::default()).contains("choice")); + w.focus(Field::Device); + assert!(render(&w, &State::default()).contains("read only")); + w.focus(Field::Host); + assert!(render(&w, &State::default()).contains("editable")); + // Port names the direction it would move in, not a generic pair. + w.focus(Field::Port); + assert!(render(&w, &State::default()).contains("choice · ←→ switch to custom")); + w.port_mode = PortMode::Custom; + assert!( + render(&w, &State::default()) + .contains("choice · ←→ back to automatic · Enter to edit the port") + ); + w.focus(Field::Host); + let host = w.host.clone(); + w.editor = Some(InlineEditor::new(Field::Host, &host)); + assert!(render(&w, &State::default()).contains("Esc cancel")); + } + + #[test] + fn no_generic_change_footer_remains() { + let w = ServeWizardState::default(); + assert!(!render(&w, &State::default()).contains("←→ change")); } #[test] @@ -981,11 +2301,12 @@ mod tests { let mut wiz = Some(ServeWizardState::default()); let mut jobs = State::default(); wiz.as_mut().unwrap().model = "qwen".into(); - wiz.as_mut().unwrap().field = FIELDS.iter().position(|f| *f == Field::Launch).unwrap(); + wiz.as_mut().unwrap().focus(Field::Launch); on_key(&mut wiz, &mut jobs, &[], key(KeyCode::Enter)); let out = render(wiz.as_ref().unwrap(), &jobs); assert!(out.contains("Review:"), "approval modal shown"); assert!(out.contains("serve"), "describes the gated launch"); assert!(out.contains("Approve"), "approve button present"); + assert!(out.contains("endpoint shown after launch"), "{out:?}"); } } diff --git a/crates/rocm-dash-tui/src/ui/tabs/chat.rs b/crates/rocm-dash-tui/src/ui/tabs/chat.rs index 8843763e1..d955c0da1 100644 --- a/crates/rocm-dash-tui/src/ui/tabs/chat.rs +++ b/crates/rocm-dash-tui/src/ui/tabs/chat.rs @@ -62,7 +62,7 @@ fn draw_consent(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { ( " Chat — detecting… ", vec![Line::from(Span::styled( - "Probing for a local engine (Lemonade :13305 / vLLM :8000 / rocm serve :11435)…", + "Probing for a local engine (Lemonade :13305 / vLLM :8000 / a rocm serve service)…", Style::default().fg(theme.fg), ))], ) @@ -211,7 +211,7 @@ fn consent_gate_lines<'a>( Style::default().fg(theme.muted), )), Line::from(Span::styled( - " • or run a local endpoint (vLLM :8000, rocm serve :11435)", + " • or run a local endpoint (vLLM :8000, a rocm serve service)", Style::default().fg(theme.muted), )), ], @@ -637,7 +637,10 @@ mod tests { assert!(render_str(&mut s).contains("detect a local engine")); // After a fruitless detect, the message shows. s.set_detect_result(None); - assert!(render_str(&mut s).contains("no local engine found")); + let out = render_str(&mut s); + assert!(out.contains("no local engine found")); + // `rocm serve` leases its port at launch, so no fixed one is promised. + assert!(!out.contains("11435"), "{out:?}"); } #[test] @@ -645,7 +648,9 @@ mod tests { let mut s = AppState::new("t".into(), "default-dark".into()); s.active_tab = crate::app::ActiveTab::Chat; s.request_detect(); - assert!(render_str(&mut s).contains("Probing for a local engine")); + let out = render_str(&mut s); + assert!(out.contains("Probing for a local engine")); + assert!(!out.contains("11435"), "{out:?}"); } #[test] diff --git a/crates/rocm-dash-tui/src/ui/tabs/serving.rs b/crates/rocm-dash-tui/src/ui/tabs/serving.rs index 8ce64b34c..11fc01b1f 100644 --- a/crates/rocm-dash-tui/src/ui/tabs/serving.rs +++ b/crates/rocm-dash-tui/src/ui/tabs/serving.rs @@ -25,11 +25,11 @@ pub const VERBS: &[Verb] = &[ summary: "Launch a model on a serving engine and expose an OpenAI-style endpoint.", steps: &[ "Pick a model", - "Choose an engine — Lemonade · vLLM", - "Set GPU placement (required / preferred / CPU-only)", - "Launch on 127.0.0.1:11435 and watch it come up", + "Leave the rest automatic, or open Advanced settings", + "GPU is required: ROCm never falls back to CPU", + "Launch on a local endpoint picked for you and watch it come up", ], - cmd: "rocm serve --engine …", + cmd: "rocm serve ", read_only: false, badge: None, }, @@ -156,6 +156,28 @@ mod tests { .collect() } + #[test] + fn serving_first_view_promises_no_concrete_endpoint() { + // The CLI leases the local port at launch time, so the first thing a + // user reads must not promise a fixed `127.0.0.1:11435` endpoint. + let mut s = AppState::new("t".into(), "default-dark".into()); + s.active_tab = ActiveTab::Serving; + s.serving_sel = 0; // "Serve a model" + let out = render(&s, 200, 30); + assert!( + !out.contains("11435"), + "first-view copy must not promise a fixed port: {out:?}" + ); + assert!( + !out.contains("127.0.0.1"), + "first-view copy must not promise a fixed host: {out:?}" + ); + assert!( + out.contains("Launch on a local endpoint picked for you"), + "automatic endpoint wording missing: {out:?}" + ); + } + #[test] fn serving_renders_its_six_rows() { let mut s = AppState::new("t".into(), "default-dark".into()); diff --git a/docs/architecture.md b/docs/architecture.md index 937d9f312..d0ef598cc 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -48,7 +48,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`freeform_planner.rs`](../apps/rocm/src/freeform_planner.rs) — full domain extraction (Phase 6e, ROCMAI-87). Owns the freeform natural-language planner cluster's types (`StructuredRequestPlan`, `PlannerIntent`, `PlannedToolCall`, `FreeformPlanAction`, `ProviderPlannerResponse`, `ProviderPlannerToolCall`, all module-private except `FreeformPlanAction`, `StructuredRequestPlan` and `PlannedToolCall`, which are `pub(crate)`), the deterministic plan builder (`build_freeform_plan*`), the provider-assisted path (`resolve_freeform_plan_with_provider`, `build_provider_planner_prompt`, `provider_planner_response_to_plan`, `validate_provider_planner_tool_call`), and plan rendering (`render_structured_request_plan`). [`main.rs`](../apps/rocm/src/main.rs) calls `build_freeform_plan_with_context`/`render_structured_request_plan` directly for the dispatch/review flow ([`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs)'s `render_launch_summary` → `host_default_engine` is another call into this module, unrelated to planning: bare `rocm`'s non-interactive launch summary), while [`freeform_execution.rs`](../apps/rocm/src/freeform_execution.rs) above calls `plan_next_action`/`format_structured_tool_call` for the execution flow and reads `FreeformPlanAction`/`StructuredRequestPlan` by type for its execution-validation functions — those live there, not here, because they decide whether a plan's next action is safe to *execute*, a dispatch-adjacent concern distinct from *building* the plan; `main.rs`'s `run_freeform` builds a plan exactly once via `build_freeform_plan_with_context` and threads that same value through `execute_freeform_next_action` to `prepare_freeform_execution` rather than rebuilding it (ROCMAI-725: building a plan can call a configured cloud `planner_provider`, which is not guaranteed deterministic, so a second independent build could silently disagree with the first). [`chat.rs`](../apps/rocm/src/chat.rs)'s `natural_language_plan` MCP tool gets the same single-build guarantee from `plan_freeform_request_with_context`, which builds once and returns both the rendered text and the next action; `chat.rs` also calls `format_structured_tool_call` directly, like [`comfyui.rs`](../apps/rocm/src/comfyui.rs) does, and its `validate_chat_rocm_command_safety` reads `serve_args_request_cpu_device` directly as a no-CPU-fallback safety check on provider-proposed `serve` args. `host_default_engine`, `build_freeform_plan_with_context`, `render_structured_request_plan`, `plan_next_action`, `plan_freeform_request_with_context`, `provider_planner_response_to_plan`, `format_structured_tool_call`, and `serve_args_request_cpu_device` are `pub(crate)` for these reasons, as are `infer_engine_from_request` (read from `chat.rs`) and `plan_driver_request` (read from [`driver_install.rs`](../apps/rocm/src/driver_install.rs)'s test module); `freeform_plan_next_action` and `render_freeform_plan` are test-only (`#[cfg(test)]`) and `FreeformPlanAction` was already `pub(crate)` before the split. `parse_freeform_invocation`/`should_treat_as_freeform` (deciding whether an invocation is freeform at all) stay in `main.rs`, a dispatch-level concern outside both this module and `freeform_execution.rs`. The cluster's regression tests live in `freeform_planner.rs`'s own `mod tests`; the handful of `main.rs` tests that exercise the invocation classifier/clap-suggestion-avoidance stay in `main.rs`'s shared `mod tests`, since neither is specific to this module or to `freeform_execution.rs` (and neither touches `freeform_planner::` at all), and the ROCMAI-725 regression tests proving the single-build guarantee live in `providers.rs`'s `mod tests` (reusing its mock local-provider server), reaching this module's items through the qualified `freeform_planner::` path like any other external caller. - [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-214): `--device`/`--gpu` parsing (`parse_device_policy`, `parse_gpu_selection`, `parse_gpu_indices_arg`), the managed-launch-lock GPU selection (`select_gpu_indices_under_launch_lock`, `validate_pinned_gpu_index`, `effective_gpu_count`, `auto_select_gpu_indices`, `select_auto_gpu_index`, `busy_gpu_indices`), per-GPU VRAM telemetry (the `GpuVramUsage` type, `gpu_vram_usage`, `simulated_low_vram_usage`, `gpu_vram_usage_amd_smi`, `gpu_vram_usage_sysfs`, `read_drm_vram_usage`, `read_sysfs_u64`, `drm_device_is_amd`, `parse_gpu_vram_usage`), and the serve-plan low-VRAM warning (`gpu_low_memory_warning`, `vram_capacity_is_meaningful`, `serve_gpu_low_memory_warning`), plus the best-effort `amd-smi`-based GPU count (`detect_gpu_count`). `const fn device_policy_name` sits immediately after this cluster's last function in the original file but stayed in `main.rs` rather than moving — it already had callers outside the cluster (`engine_ipc.rs`, `service_launch.rs`, `serve_cmd.rs`) plus its own test. `GpuVramUsage` (and its fields), `gpu_vram_usage`, `parse_gpu_indices_arg`, `serve_gpu_low_memory_warning`, and `vram_capacity_is_meaningful` are `pub(crate)` and read from [`main.rs`](../apps/rocm/src/main.rs) (`classify_accelerator_memory`, `host_accelerator_memory`, the `dispatch()` `Command::EngineServeHttp` arm, and `collect_serve_notes`); `detect_gpu_count`, `gpu_vram_usage`, `parse_device_policy`, `parse_gpu_selection`, `select_gpu_indices_under_launch_lock`, `serve_gpu_low_memory_warning`, and `validate_pinned_gpu_index` are also `pub(crate)` and read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs); `parse_device_policy` is also read from [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs)'s `run_builtin_engine_serve_http` and, as of Phase 6f (ROCMAI-209), from [`services_text.rs`](../apps/rocm/src/services_text.rs)'s `restart_child_argv` (moved out of `main.rs` in that same phase). `effective_gpu_count`, `auto_select_gpu_indices`, `select_auto_gpu_index`, `simulated_low_vram_usage`, `gpu_vram_usage_amd_smi`, `gpu_vram_usage_sysfs` (both the Linux and non-Linux `cfg` variants), `read_drm_vram_usage`, `read_sysfs_u64`, `drm_device_is_amd`, `parse_gpu_vram_usage`, `gpu_low_memory_warning`, and `busy_gpu_indices` have no callers outside this module and stay private. The cluster's regression tests live in `gpu_policy.rs`'s own `mod tests`; two small test-only fixture builders (`vram`, `host_gpu`) are duplicated rather than shared, since `main.rs`'s own `serve_notes_pair_low_vram_with_the_vllm_utilization_hint` (testing `collect_serve_notes`, which stayed) still needs them too. - [`logging.rs`](../apps/rocm/src/logging.rs) — mechanical relocation, no owned types, reached by qualified path. -- [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-211): managed-service liveness classification (`managed_service_is_live`, `refresh_managed_service_runtime_liveness`, `existing_live_managed_service`'s idempotency guard) and the dashboard-sidebar status/counts helpers derived from it (`ManagedServiceSidebarCounts`/`managed_service_sidebar_counts`/`local_server_sidebar_status`), plus the engine display-name helper `friendly_engine_label`, read only by [`chat.rs`](../apps/rocm/src/chat.rs). `ManagedServiceSidebarCounts` is `pub(crate)` to match the visibility of the `pub(crate)` functions that return/take it; `friendly_engine_label`, `existing_live_managed_service`, `recorded_service_pids`, `refresh_managed_service_runtime_liveness`, `local_server_sidebar_status`, `managed_service_sidebar_counts`, and `load_managed_services` are `pub(crate)` for the same reason — each has a confirmed caller outside this module. [`main.rs`](../apps/rocm/src/main.rs) uses its liveness helpers (`managed_service_is_live`, `recorded_service_pids`) for the in-use check behind service-record removal and pruning, and `load_managed_services` for prune planning and the codex-bridge snapshot; [`service_launch.rs`](../apps/rocm/src/service_launch.rs) is the only caller of the `existing_live_managed_service` duplicate-launch guard; [`services_text.rs`](../apps/rocm/src/services_text.rs) reads the sidebar helpers; other modules, including [`chat.rs`](../apps/rocm/src/chat.rs) and [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs), read the registry and liveness helpers. +- [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-211): managed-service liveness classification (`managed_service_is_live`, `refresh_managed_service_runtime_liveness`, `existing_live_managed_service_in`'s idempotency guard) and the dashboard-sidebar status/counts helpers derived from it (`ManagedServiceSidebarCounts`/`managed_service_sidebar_counts`/`local_server_sidebar_status`), plus the engine display-name helper `friendly_engine_label`, read only by [`chat.rs`](../apps/rocm/src/chat.rs). `ManagedServiceSidebarCounts` is `pub(crate)` to match the visibility of the `pub(crate)` functions that return/take it; `friendly_engine_label`, `existing_live_managed_service_in`, `recorded_service_pids`, `refresh_managed_service_runtime_liveness`, `local_server_sidebar_status`, `managed_service_sidebar_counts`, and `load_managed_services` are `pub(crate)` for the same reason — each has a confirmed caller outside this module. [`main.rs`](../apps/rocm/src/main.rs) uses its liveness helpers (`managed_service_is_live`, `recorded_service_pids`) for the in-use check behind service-record removal and pruning, and `load_managed_services` for prune planning and the codex-bridge snapshot; [`service_launch.rs`](../apps/rocm/src/service_launch.rs) is the only caller of the `existing_live_managed_service_in` duplicate-launch guard; [`services_text.rs`](../apps/rocm/src/services_text.rs) reads the sidebar helpers; other modules, including [`chat.rs`](../apps/rocm/src/chat.rs) and [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs), read the registry and liveness helpers. - [`provider_keys.rs`](../apps/rocm/src/provider_keys.rs) — full domain extraction. - [`providers.rs`](../apps/rocm/src/providers.rs) — full domain extraction (no dedicated command enum; invoked from the existing chat/config command flows). - [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) — mechanical relocation, owns types (part of [`main.rs`](../apps/rocm/src/main.rs)'s Phase 6c split, ROCMAI-89, alongside [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) below): settling an engine onto a runtime after an SDK install — torch alignment, engine-dependency checks, and device checks. `SdkInstallConsents` is `pub(crate)` and read from `main.rs`'s `install()`; `ensure_libatomic_for_torch`/`ensure_libnuma_for_torch` are also `pub(crate)` and called from both `main.rs`'s `install()` and [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs); `ensure_openmpi_for_vllm`/`settle_engine_install` are `pub(crate)` and read from `engines_cmd.rs`; `real_os_release` is `pub(crate)` and read from [`driver_install.rs`](../apps/rocm/src/driver_install.rs). @@ -56,8 +56,9 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) — mechanical relocation, owns types (Phase 6c, ROCMAI-89): `runtimes()` and its activate/rollback/uninstall/import/adopt handlers, plus runtime manifest validation/registry. `RuntimesCommand` stayed in [`main.rs`](../apps/rocm/src/main.rs) (part of the `Command`/`Cli` clap definition, the same way `EnginesCommand` did in Phase 6a). `RuntimeActivationResult` and `ActiveRuntimeMarker` are `pub(crate)`, read from [`update_cmd.rs`](../apps/rocm/src/update_cmd.rs)'s `apply_runtime_update`/`append_update_activate_summary` and from [`storage.rs`](../apps/rocm/src/storage.rs) respectively; `select_runtime_manifest`/`validate_runtime_manifest_for_activation`/`normalize_path_for_compare`/`recover_setup_runtime_registration`/`runtime_usability_status` are also `pub(crate)`, read from other modules such as [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs) and [`comfyui.rs`](../apps/rocm/src/comfyui.rs). The dependency on the sibling [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) above runs the other way: that module calls into this one (`activate_runtime`, `newest_installed_runtime_manifest`, `recover_setup_runtime_registration`), while this module reaches it only from its test module. - [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) — mechanical relocation, owns types (Phase 6a): `serve()`, plus engine-recipe overrides. - [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs) — full domain extraction. +- [`serve_port.rs`](../apps/rocm/src/serve_port.rs) — full domain extraction: the central `rocm serve` port policy (automatic vs explicit request, reservation by live managed services, OS preflight lease), used by [`service_launch.rs`](../apps/rocm/src/service_launch.rs); the CLI restart path shares the `rocm-core` lease primitives (`lease_loopback_port_or_bind_error`, `is_canonical_loopback_host`) rather than this module. - [`service_hints.rs`](../apps/rocm/src/service_hints.rs) — shared CLI-output component: the `rocm services stop|restart|remove --yes` hint text, shared by [`main.rs`](../apps/rocm/src/main.rs), [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs), [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs), [`service_launch.rs`](../apps/rocm/src/service_launch.rs), and [`services_text.rs`](../apps/rocm/src/services_text.rs) so they cannot go out of sync with each other, as `serve_summary.rs`'s stop row did from the day it was added — #417. -- [`service_launch.rs`](../apps/rocm/src/service_launch.rs) — mechanical relocation, owns types (Phase 6b, ROCMAI-91): spawning/attaching to a managed service process and streaming its log, plus the endpoint-key/auth checks that gate a public bind — `AttachOutcome`/`RawModeGuard`/`ManagedSpawn`/`AutostartClaim` are its private types; `ManagedLaunchReport` is `pub(crate)`, since [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) reads its fields directly for the interactive summary. Most of this module's functions are `pub(crate)` for the same reason — other modules, including [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), [`automations.rs`](../apps/rocm/src/automations.rs), and [`services_text.rs`](../apps/rocm/src/services_text.rs), call into it; `main.rs` does not. `ServicesCommand`/`services()` stayed in [`main.rs`](../apps/rocm/src/main.rs) rather than moving alongside — the service-record-management helpers `services()` dispatches to (stop/restart/remove/prune) are entangled with each other; only stop/restart are also reachable from the chat/LLM sandbox tool-calling path, out of scope for Phase 6b. +- [`service_launch.rs`](../apps/rocm/src/service_launch.rs) — mechanical relocation, owns types (Phase 6b, ROCMAI-91): spawning/attaching to a managed service process and streaming its log, the port allocation transaction and failed-launch cleanup (under the shared launch lock), plus the endpoint-key/auth checks that gate a public bind — `AttachOutcome`/`RawModeGuard`/`ManagedSpawn`/`AutostartClaim` are its private types; `ManagedLaunchReport` is `pub(crate)`, since [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) reads its fields directly for the interactive summary. Most of this module's functions are `pub(crate)` for the same reason — other modules, including [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), [`automations.rs`](../apps/rocm/src/automations.rs), and [`services_text.rs`](../apps/rocm/src/services_text.rs), call into it; `main.rs` does not. `ServicesCommand`/`services()` stayed in [`main.rs`](../apps/rocm/src/main.rs) rather than moving alongside — the service-record-management helpers `services()` dispatches to (stop/restart/remove/prune) are entangled with each other; only stop/restart are also reachable from the chat/LLM sandbox tool-calling path, out of scope for Phase 6b. - [`services_text.rs`](../apps/rocm/src/services_text.rs) — mechanical relocation, owns private types (Phase 6f, ROCMAI-209): the CLI/TUI logs browser (`render_logs_text`, `render_logs_browser_text`, `render_logs_browser_page_text`, `render_logs_browser_page_text_with_options`, `logs_browser_total_pages`, the `LogBrowserEntry` type, `collect_log_browser_entries`, `filter_log_browser_entries`, `log_browser_source_label`, `write_past_attempts_hint`), the `rocm services`/`rocm services logs` status surfaces (`render_services_text`, `render_services_json`, `render_service_logs_text`, `render_service_logs_text_with_options`, `render_service_action_result`, `read_managed_service_manifest`, `load_managed_service`), the internal `rocmd`-compatible status/sandbox surface (`render_internal_status_text`, `run_internal_sandbox_tool`), and the managed-service stop/restart implementation behind both the sandbox tool and `rocm services stop|restart` (`terminate_recorded_service_pids`, the `MANAGED_STOP_GRACE` constant, `stop_internal_managed_service`, `stop_internal_managed_service_with`, `stop_internal_managed_service_confirmed_with`, `unload_lemonade_service_model`, `apply_restart_runtime_pin`, `load_service_for_restart`, `restart_child_argv`, `restart_internal_managed_service`, `restart_internal_managed_service_with`, `update_restarting_record`, `record_restarted_child`, `record_restart_readiness`), plus `validate_service_id`, `list_log_files`, `read_optional_tail_lines`, and the CLI-lifecycle-log tail formatting helpers (`format_cli_lifecycle_tail_line`, `lifecycle_event_label`, `humanize_log_label`, `lifecycle_field`, `read_tail_lines`). Two items physically interleaved in this span in `main.rs` moved with it rather than staying behind, since neither has a caller outside the cluster: the `LogBrowserEntry` struct and the `MANAGED_STOP_GRACE` constant; both stay private. `dispatch()` calls directly into this module at several points — `render_internal_status_text`, `run_internal_sandbox_tool`, `render_service_logs_text`, `render_logs_browser_text`, and `render_logs_text` — and `main.rs`'s `services()` calls `render_services_text`, `render_services_json` and `render_service_logs_text`, all re-imported via `use crate::services_text::{...};`. `render_service_action_result`, `run_internal_sandbox_tool`, and `validate_service_id` are also `pub(crate)` and read from `main.rs`'s own `run_approved_service_action` (part of the broader service-record-management cluster that stays at the crate root per [`service_launch.rs`](../apps/rocm/src/service_launch.rs)'s entry above); `validate_service_id` is also read from `main.rs`'s `remove_managed_service_record`/`service_record_artifacts`, and `load_managed_service` from its `remove_managed_service_record`/`apply_service_prune_plan`. `load_managed_service` and `restart_internal_managed_service` are `pub(crate)`, read from [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs) (reconciling live servers against a runtime activation) and from [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs)'s own test module; `read_tail_lines`, `render_logs_text`, `render_logs_browser_text`, `render_services_text`, `render_service_logs_text`, and `validate_service_id` are also `pub(crate)`, read from [`chat.rs`](../apps/rocm/src/chat.rs)'s deterministic tool-output summaries; `read_optional_tail_lines` and `render_service_action_result` are also `pub(crate)`, read from [`service_launch.rs`](../apps/rocm/src/service_launch.rs)'s managed-spawn/attach path. Every other item in this list has no caller outside this module and stays private, including `restart_internal_managed_service_with`, `restart_child_argv`, `load_service_for_restart`, `apply_restart_runtime_pin`, and `unload_lemonade_service_model` — see [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs)'s and [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs)'s entries above for how those two modules' own doc comments now describe this module, rather than `main.rs`, as that caller. The cluster's regression tests live in this module's own `mod tests`, reaching shared fixtures (`test_paths`, `plant_service_record_on_runtime`, `OLD_RUNTIME_KEY`, `NEW_RUNTIME_KEY`) through `crate::tests::` the same way other mechanically relocated modules do, and `runtime_services::` through its own `use crate::runtime_services;` for the handful of tests that drive this cluster's restart/stop behavior through `runtime_services`'s public entry points rather than calling it directly. Three tests that call this cluster's `load_managed_service`/`render_service_action_result` only incidentally — their primary subject is `runtime_services::pin_service_record_to_runtime`/`restore_service_record_pin`, `runtime_services::restart_service_onto_runtime_with`'s outcome classification, and `main.rs`'s own `run_approved_service_action` respectively — stay in `main.rs`'s shared `mod tests` instead. - [`storage.rs`](../apps/rocm/src/storage.rs) — full domain extraction. - [`therock.rs`](../apps/rocm/src/therock.rs) — full domain extraction; owns TheRock SDK install/update and the runtime-manifest model/loaders. The `runtimes` subcommand's command enum (`RuntimesCommand`) stays in [`main.rs`](../apps/rocm/src/main.rs); its dispatch fn and most of its domain logic — rendering, activation, and manifest import — live in [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs). diff --git a/docs/manual-testing.md b/docs/manual-testing.md index ed8f7da68..e7cf2f660 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -320,6 +320,34 @@ While the log stream is attached, verify detach and stop behave differently: - Re-run the serve command and press `Ctrl+C` instead. The server shuts down and `rocm services` no longer lists it as running. +Omit `--port` on the default loopback host to exercise automatic selection: + +```powershell +rocm serve qwen --engine lemonade --device gpu_required --managed +``` + +Expected result: + +- The command picks the first free port in `11435`-`11535` instead of failing + or always landing on `11435`. +- `rocm services` shows the launched service bound to the port it picked, and + the deployment summary's endpoint matches it. + +With that service still running, try to launch a service for a different +model pinned to the same port. (The same model and engine would be reported as +already running instead, since an equivalent live service is reused before any +port check.) + +```powershell +rocm serve --engine lemonade --device gpu_required --managed --port +``` + +Expected result: + +- The command refuses up front, naming the service already holding that port + and how to free it (`rocm services stop --yes`), without + spawning an engine process or writing a new record. + ### Windows display driver check On a Windows machine with an AMD GPU, check what the CLI says about the display diff --git a/docs/testing.md b/docs/testing.md index e49a3c524..a58d32108 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -951,6 +951,23 @@ cargo test -p rocmd recovery_respawn_fails_closed cargo test -p rocmd apply_endpoint_key_env ``` +Collision-safe port selection, focused tests: + +```bash +cargo test -p rocm --bin rocm -- serve_port:: +cargo test -p rocm --bin rocm automatic_selects_against_real_loopback_listeners +cargo test -p rocm --bin rocm concurrent_auto_transactions_publish_distinct_ports +cargo test -p rocm --bin rocm concurrent_equivalent_transactions_converge_on_one_record +cargo test -p rocmd recovery_waits_for_the_allocation_lock_and_refuses_an_occupied_port +``` + +Omitting `--port` on the default loopback host scans `11435..=11535` for a +candidate neither reserved by a live managed-service record nor occupied on the +OS ([`serve_port.rs`](../apps/rocm/src/serve_port.rs)), under the same `services/launch.lock` that serializes +every other managed launch and `rocmd` recovery attempt. An explicit `--port` +on a non-default `--host` is still required: automatic selection only probes +`127.0.0.1` and refuses to guess elsewhere. + A loopback bind (`rocm serve `, default `127.0.0.1`) stays credential-free. A public bind (`--host 0.0.0.0 --allow-public-bind`) requires an API key: pass one with `--api-key` / the `ROCM_SERVE_API_KEY` environment diff --git a/skills/rocm-cli-assistant/SKILL.md b/skills/rocm-cli-assistant/SKILL.md index 66af24a85..a386d6fa1 100644 --- a/skills/rocm-cli-assistant/SKILL.md +++ b/skills/rocm-cli-assistant/SKILL.md @@ -28,6 +28,7 @@ Use this skill when answering ROCm CLI local assistant questions. - The built-in assistant is fixed to qwen served by Lemonade with GPU required. Do not switch the built-in assistant to vLLM. - Installing an engine and running a model server are different states. Answer each separately when the user asks. - `rocm serve` accepts `--gpu auto|` to pick the AMD GPU. `auto` (default) prefers a GPU that looks idle from VRAM telemetry (`amd-smi`, or the amdgpu DRM sysfs counters when `amd-smi` is absent) and is not already used by another rocm-cli server (managed or foreground), falling back to the GPU with the most free memory; a single index pins one GPU. Serving one model across multiple GPUs is not supported. Do not suggest CPU fallback when a GPU is busy or out of range. +- Automatic port selection only works on host `127.0.0.1` exactly. `localhost` and `::1` are loopback for auth purposes but still need an explicit `--port` when launching a server, or the call is refused. - Which engines actually run on the user's machine is stated in the host facts the CLI appends to this prompt. Answer from those facts; do not assume a platform. Never suggest CPU fallback. - On native Windows, vLLM serving/install live checks are skipped; tell the user to use WSL/Linux for that ROCm GPU engine and do not suggest CPU fallback. diff --git a/tests/e2e-cucumber/features/dash.feature b/tests/e2e-cucumber/features/dash.feature index 64fd01ee4..37732aeb0 100644 --- a/tests/e2e-cucumber/features/dash.feature +++ b/tests/e2e-cucumber/features/dash.feature @@ -667,3 +667,85 @@ Feature: Interactive dashboard Then the warning text is no longer displayed When the user quits the dashboard Then the dashboard exits successfully + + @id:dash-serve-wizard-opens-collapsed @requires-os:linux + Scenario: dash-39 - Opening the serve wizard shows only the model, Advanced settings, and Launch + When the user opens the dashboard with demo data + And the user opens the Observe view + And the user opens the serve wizard + Then the serve wizard shows only the model, Advanced settings, and Launch + When the user presses Escape + Then the serve wizard is closed + When the user quits the dashboard + Then the dashboard exits successfully + + @id:dash-serve-wizard-escape-cancels-edit-without-closing @requires-os:linux + Scenario: dash-40 - Escape cancels an in-progress host edit without closing the serve wizard + When the user opens the dashboard with demo data + And the user opens the Observe view + And the user opens the serve wizard + And the user expands the serve wizard's Advanced settings + And the user starts editing the serve wizard's Host field + And the user types "nope" into the serve wizard + And the user presses Escape + Then the serve wizard is still open with the host unchanged + When the user presses Escape + Then the serve wizard is closed + When the user quits the dashboard + Then the dashboard exits successfully + + @id:dash-serve-wizard-automatic-form-stages-bare-serve @requires-os:linux + Scenario: dash-41 - An automatic serve form stages a bare serve command with an automatic port + When the user opens the dashboard with demo data + And the user opens the Observe view + And the user opens the serve wizard + And the user types "demo-model" into the serve wizard + And the user presses Down 2 times in the serve wizard + And the user presses Enter in the serve wizard + Then the serve wizard asks to approve "serve demo-model" with an automatic port + When the user presses Escape + Then the serve wizard approval closes and the wizard stays open + When the user presses Escape + Then the serve wizard is closed + When the user quits the dashboard + Then the dashboard exits successfully + + @id:dash-serve-wizard-refuses-custom-host-with-automatic-port @requires-os:linux + Scenario: dash-42 - A custom host with an automatic port is refused in the form + When the user opens the dashboard with demo data + And the user opens the Observe view + And the user opens the serve wizard + And the user types "demo-model" into the serve wizard + And the user presses Down 1 times in the serve wizard + And the user presses Enter in the serve wizard + And the user presses Down 3 times in the serve wizard + And the user presses Enter in the serve wizard + And the user types "localhost" into the serve wizard + And the user presses Enter in the serve wizard + And the user presses Down 3 times in the serve wizard + And the user presses Enter in the serve wizard + Then the serve wizard refuses to launch and says "set Port to Custom" + When the user presses Escape + Then the serve wizard is closed + When the user quits the dashboard + Then the dashboard exits successfully + + @id:dash-serve-wizard-refuses-public-host @requires-os:linux + Scenario: dash-43 - A public host is refused in the form with the CLI pointer + When the user opens the dashboard with demo data + And the user opens the Observe view + And the user opens the serve wizard + And the user types "demo-model" into the serve wizard + And the user presses Down 1 times in the serve wizard + And the user presses Enter in the serve wizard + And the user presses Down 3 times in the serve wizard + And the user presses Enter in the serve wizard + And the user types "0.0.0.0" into the serve wizard + And the user presses Enter in the serve wizard + And the user presses Down 3 times in the serve wizard + And the user presses Enter in the serve wizard + Then the serve wizard refuses to launch and says "Dash serves on loopback only" + When the user presses Escape + Then the serve wizard is closed + When the user quits the dashboard + Then the dashboard exits successfully diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 25fed56b7..65b717bfc 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -323,6 +323,28 @@ Feature: Model serving Then the CLI refuses and names the exact command to stop the existing service And running that command actually stops the service + # An explicit --port is a demand: when an equivalent service (same engine and + # model) is already live on a different port, serve refuses instead of + # reporting success on the old port. Same planted record and lane as serve-24. + @id:serve-explicit-port-refused-for-live-equivalent @requires-gpu @requires-engine:lemonade @requires-os:linux + Scenario: serve-32 - An explicit port that differs from a running equivalent service is refused + Given a managed runtime is active + And an unauthenticated Lemonade service is already running for that model + When the user serves that same model again on an explicit port other than the running one + Then the CLI refuses and names the port the running service uses + And the CLI names the exact command to stop the existing service + + # `rocm services restart` takes the launch lock and refuses a port another + # live managed service now holds, before it spawns anything. No GPU: the + # refusal comes from the on-disk records. + @id:services-restart-refused-when-port-claimed @requires-os:linux + Scenario: serve-33 - Restarting a service whose port another service holds is refused + Given a managed runtime is active + And another managed service already reserves the default serve port + And a stopped managed service is recorded on the default serve port + When the user restarts the stopped service + Then the restart is refused naming the service that holds the port + # ROCMAI-543: a Windows console on the OEM code page (437) renders a UTF-8 em # dash, and other non-ASCII punctuation, as mojibake (e.g. "ΓÇö") instead of # transliterating it. `rocm model` is the flagship instance of that bug class; @@ -379,3 +401,61 @@ Feature: Model serving And the user sends a chat completion request Then the response contains a model reply And the response identifies the correct model + + # Parse-time refusal (serve-port rewrite): an out-of-range `--port` (0 here; + # 65536 and non-numeric values share the same parser and message) is rejected + # before any engine or GPU work, so it needs no GPU and gates every PR + # (ungated), like its sibling serve-14. + @id:serve-invalid-port-rejected + Scenario: serve-28 - Serving with an out-of-range port is refused with a clear reason + When the user serves a model with an out-of-range port + Then serving is refused before any engine starts + And the user is told the port must be between 1 and 65535 + + # Collision-safe explicit ports (serve-port rewrite). An explicit `--port` + # equal to one a live managed service already holds must be refused before any + # bind is attempted, naming the service so the user can stop it or pick + # another port. Reaching this check needs a real GPU: `resolve_serve_port` + # only runs once `spawn_managed_engine_child` has a completed model/engine + # resolution in hand, and `serve` has no CPU fallback that gets there. The + # "stop the service holding that port" advice is checked here by its wording + # only; the advice is actually proven by + # `stopping_the_reserving_record_frees_its_explicit_port_for_reuse` in + # apps/rocm/src/service_launch.rs, per AGENTS.md section 3. + # Runs in the merge queue, like its automatic-port sibling serve-auto-port-skips-occupied. + @id:serve-explicit-port-reserved @requires-gpu @merge-queue + Scenario: serve-29 - Serving on a port another managed service already holds is refused + Given a managed runtime is active + And another managed service already reserves the default serve port + When the user serves a model explicitly on the reserved port + Then serving is refused before any engine starts + And the user is told to stop the service holding that port + + # Collision-safe explicit ports, untracked-process case (serve-port rewrite, + # sibling of serve-29). No managed-service record exists here, so + # `resolve_serve_port` never reaches the "reserved by managed service" branch + # and instead gets a plain OS-level `AddrInUse` back from the bind probe, + # routing to `explicit_bind_error`'s "already in use by another process" + # message. Needs the same `@requires-gpu` as serve-29 for the same reason: + # `resolve_serve_port` only runs once `spawn_managed_engine_child` has a + # completed model/GPU resolution in hand (GPU indices are selected earlier in + # `serve_cmd.rs`, under the launch lock, before any port is ever probed), and + # `serve` has no CPU fallback that gets there. + @id:serve-explicit-port-in-use-by-untracked-process @requires-gpu @merge-queue + Scenario: serve-30 - Serving on a port an untracked process already holds is refused + Given a managed runtime is active + And the default serve port is occupied + When the user serves a model explicitly on the occupied port + Then serving is refused before any engine starts + And the user is told the port is already in use by another process + + # Real CLI coverage for automatic port allocation. A loopback listener holds + # the legacy default while one real managed GPU serve starts; the reported + # endpoint must advance to the next candidate rather than failing late in the + # engine. Runs in the merge queue because it launches a real model server. + @id:serve-auto-port-skips-occupied @requires-gpu @merge-queue + Scenario: serve-31 - Automatic serving skips an occupied default port + Given a managed runtime is active + And the default serve port is occupied + When the user serves a model with automatic port selection + Then the service uses the next automatic port diff --git a/tests/e2e-cucumber/features/networking.feature b/tests/e2e-cucumber/features/networking.feature index fe16f4075..7282ba394 100644 --- a/tests/e2e-cucumber/features/networking.feature +++ b/tests/e2e-cucumber/features/networking.feature @@ -39,3 +39,16 @@ Feature: Native HTTP networking When the user serves a model bound to a public interface without allowing public binding Then serving is refused before any engine starts And the user is told to allow public binding first + + # Collision-safe ports (serve-port rewrite). Automatic port selection proves a + # port is free by probing it on loopback before the engine starts; that proof + # only applies to the canonical default host (127.0.0.1), so any other host + # name (even a loopback alias like `localhost` that needs no public-bind + # opt-in) must come with an explicit `--port`. This check runs in + # `validate_port_request`, before any engine or GPU work, so it needs no GPU + # and runs on the mock lane every PR. + @id:networking-custom-host-needs-explicit-port + Scenario: networking-04 - A non-canonical host without an explicit port is refused up front + When the user serves a model bound to a non-canonical host without an explicit port + Then serving is refused before any engine starts + And the user is told to pass an explicit port for that host diff --git a/tests/e2e-cucumber/src/http_server.rs b/tests/e2e-cucumber/src/http_server.rs index 8ebed5736..c72a385b8 100644 --- a/tests/e2e-cucumber/src/http_server.rs +++ b/tests/e2e-cucumber/src/http_server.rs @@ -76,7 +76,20 @@ impl Drop for ServerHandle { /// For servers started from async code that then yields normally (`.await`s) /// while the server is in use. pub async fn spawn(app: Router) -> ServerHandle { - let listener = bind().await; + spawn_on(app, 0).await +} + +/// Like [`spawn`], but binds a caller-specified port. +/// +/// For scenarios that plant a managed-service record naming an exact port and +/// need a real listener behind it (e.g. so the CLI's liveness probe doesn't +/// demote the record before an explicit-port collision check runs). +pub async fn spawn_on_port(app: Router, port: u16) -> ServerHandle { + spawn_on(app, port).await +} + +async fn spawn_on(app: Router, port: u16) -> ServerHandle { + let listener = bind(port).await; let addr = local_addr(&listener); let (shutdown_tx, shutdown_rx) = oneshot::channel(); tokio::spawn(serve(listener, app, shutdown_rx)); @@ -103,7 +116,7 @@ pub fn spawn_on_own_thread(app: Router) -> ServerHandle { .build() .expect("failed to build test server runtime"); runtime.block_on(async move { - let listener = bind().await; + let listener = bind(0).await; // A dropped receiver means the starter gave up before the bind // completed; there is nothing left to serve. if addr_tx.send(local_addr(&listener)).is_err() { @@ -139,10 +152,10 @@ pub fn loopback_url(port: u16) -> Url { root_url(SocketAddr::from((Ipv4Addr::LOCALHOST, port))) } -async fn bind() -> TcpListener { - TcpListener::bind("127.0.0.1:0") +async fn bind(port: u16) -> TcpListener { + TcpListener::bind(("127.0.0.1", port)) .await - .expect("failed to bind test server listener") + .unwrap_or_else(|e| panic!("failed to bind test server listener on port {port}: {e}")) } fn local_addr(listener: &TcpListener) -> SocketAddr { diff --git a/tests/e2e-cucumber/src/mock_server.rs b/tests/e2e-cucumber/src/mock_server.rs index 74170ce73..71b1501e8 100644 --- a/tests/e2e-cucumber/src/mock_server.rs +++ b/tests/e2e-cucumber/src/mock_server.rs @@ -449,6 +449,35 @@ impl MockServer { Self::spawn(model_name, false).await } + /// Like [`Self::start`], but binds a caller-specified port instead of an + /// ephemeral one — for scenarios that plant a managed-service record + /// naming an exact port and need a real listener behind it, so the CLI's + /// liveness probe reports the model as listed instead of demoting the + /// record before an explicit-port collision check ever runs. + pub async fn start_on_port(model_name: &str, port: u16) -> Self { + let last_chat_request = Arc::new(Mutex::new(None)); + let chat_paths = Arc::new(Mutex::new(Vec::new())); + let state = ServerState { + model_name: model_name.to_string(), + metrics: None, + scripted_metrics: None, + last_chat_request: Arc::clone(&last_chat_request), + chat_paths: Arc::clone(&chat_paths), + }; + let app = Router::new() + .route("/v1/models", get(handle_models)) + .route("/models", get(handle_models)) + .route("/v1/chat/completions", post(handle_chat)) + .route("/chat/completions", post(handle_chat)) + .with_state(state); + Self { + server: http_server::spawn_on_port(app, port).await, + scripted: None, + last_chat_request, + chat_paths, + } + } + /// Like [`Self::start`], but also opts into a deterministic vLLM-flavoured /// `/metrics` route (see [`MetricsCounter`]) — for scenarios that exercise /// the dashboard's live generation-rate / TTFT / TPOT display against a diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index d0d368577..e0eecc5a0 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -65,6 +65,9 @@ pub struct E2eWorld { /// so the port keeps answering, as a started ComfyUI's does, for the rest /// of the scenario. pub comfyui_port_listener: Option, + /// Listener that deliberately occupies the default serve port while a real + /// `rocm serve` invocation proves automatic allocation advances past it. + pub automatic_port_guard: Option, /// Cache-marker destination discovered from `rocmd`'s own JSON report. pub artifact_marker_path: Option, pub endpoint: Option, @@ -299,6 +302,7 @@ impl Default for E2eWorld { artifact_server: None, paced_download_server: None, comfyui_port_listener: None, + automatic_port_guard: None, artifact_marker_path: None, endpoint: None, model_name: None, diff --git a/tests/e2e-cucumber/tests/e2e/dash_steps.rs b/tests/e2e-cucumber/tests/e2e/dash_steps.rs index 9c345b71d..cc4cb0317 100644 --- a/tests/e2e-cucumber/tests/e2e/dash_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/dash_steps.rs @@ -582,6 +582,173 @@ async fn choose_serving(world: &mut E2eWorld) { .unwrap_or_else(|e| panic!("failed to open Serving: {e}")); } +#[when("the user opens the serve wizard")] +async fn open_serve_wizard(world: &mut E2eWorld) { + // `w` opens the serve wizard, but only from the Observe tab (`actions.rs`), + // so the preceding `the user opens the Observe view` step is load-bearing + // — it also proves the event loop is reading input, the same way + // `open_services_overlay` relies on it for `s`. + let tui = session(world); + tui.send("w") + .unwrap_or_else(|e| panic!("failed to open the serve wizard: {e}")); + tui.wait_for_screen("Serve a model", default_timeout()) + .await + .unwrap_or_else(|e| panic!("serve wizard did not open: {e}")); +} + +#[then("the serve wizard shows only the model, Advanced settings, and Launch")] +async fn serve_wizard_shows_only_basic_fields(world: &mut E2eWorld) { + let tui = session(world); + tui.wait_for_screen("[ Launch ]", default_timeout()) + .await + .unwrap_or_else(|e| panic!("serve wizard did not finish drawing: {e}")); + let screen = tui.screen_text(); + assert!( + screen.contains("Model ") + && screen.contains("Advanced settings ▸ Automatic") + && screen.contains("[ Launch ]"), + "serve wizard is missing one of its collapsed-view fields:\n{screen}" + ); + assert!( + !screen.contains("Engine ") + && !screen.contains("Device ") + && !screen.contains("Host ") + && !screen.contains("Port ") + && !screen.contains("Mode "), + "serve wizard shows an advanced field while collapsed:\n{screen}" + ); +} + +// `Field::Model` is at index 0 of the collapsed `BASIC_FIELDS`, so one Down +// reaches `Field::Advanced`. Arrow keys are sent as their raw VT100 escape +// sequences, same as a real terminal would report them; the wizard's `on_key` +// only matches literal `KeyCode::Up`/`Down`, not vim-style `j`/`k`, so those +// would be typed straight into the focused field instead of moving focus. +#[when("the user expands the serve wizard's Advanced settings")] +async fn expand_serve_wizard_advanced(world: &mut E2eWorld) { + let tui = session(world); + tui.send("\x1b[B") + .unwrap_or_else(|e| panic!("failed to move to Advanced settings: {e}")); + tui.send("\r") + .unwrap_or_else(|e| panic!("failed to expand Advanced settings: {e}")); + tui.wait_for_screen("Advanced settings ▾", default_timeout()) + .await + .unwrap_or_else(|e| panic!("Advanced settings did not expand: {e}")); +} + +// With Advanced expanded, focus is still on `Field::Advanced` (index 1 of the +// full `FIELDS` order); three more Downs walk Engine → Device → Host. +#[when("the user starts editing the serve wizard's Host field")] +async fn start_editing_serve_wizard_host(world: &mut E2eWorld) { + let tui = session(world); + for _ in 0..3 { + tui.send("\x1b[B") + .unwrap_or_else(|e| panic!("failed to move toward the Host field: {e}")); + } + tui.send("\r") + .unwrap_or_else(|e| panic!("failed to start editing the Host field: {e}")); + tui.wait_for_screen("editing host", default_timeout()) + .await + .unwrap_or_else(|e| panic!("Host field editor did not open: {e}")); +} + +#[when(expr = "the user types {string} into the serve wizard")] +async fn type_into_serve_wizard(world: &mut E2eWorld, text: String) { + session(world) + .send(&text) + .unwrap_or_else(|e| panic!("failed to type {text:?} into the serve wizard: {e}")); +} + +#[then("the serve wizard is still open with the host unchanged")] +async fn serve_wizard_still_open_host_unchanged(world: &mut E2eWorld) { + let tui = session(world); + tui.wait_until_gone("editing host", default_timeout()) + .await + .unwrap_or_else(|e| panic!("Host edit was not cancelled by Escape: {e}")); + let screen = tui.screen_text(); + assert!( + screen.contains("Serve a model") + && screen.contains("[127.0.0.1]") + && !screen.contains("editing host"), + "Escape did not cleanly cancel the Host edit:\n{screen}" + ); +} + +#[when(expr = "the user presses Down {int} times in the serve wizard")] +async fn press_down_in_serve_wizard(world: &mut E2eWorld, times: u32) { + for _ in 0..times { + session(world) + .send("\x1b[B") + .unwrap_or_else(|e| panic!("failed to move down in the serve wizard: {e}")); + } +} + +#[when("the user presses Enter in the serve wizard")] +async fn press_enter_in_serve_wizard(world: &mut E2eWorld) { + session(world) + .send("\r") + .unwrap_or_else(|e| panic!("failed to press Enter in the serve wizard: {e}")); +} + +// The approval card is what the user is asked to approve, so the staged +// command line is asserted there: an automatic form stages a bare +// `serve `, with no engine or port flag the CLI should be choosing. +#[then(expr = "the serve wizard asks to approve {string} with an automatic port")] +async fn serve_wizard_asks_to_approve_automatic(world: &mut E2eWorld, command: String) { + let tui = session(world); + tui.wait_for_screen( + "Port: automatic; endpoint shown after launch", + default_timeout(), + ) + .await + .unwrap_or_else(|e| panic!("the serve wizard did not stage an approval: {e}")); + let screen = tui.screen_text(); + assert!( + screen.contains(&command) && !screen.contains("--engine") && !screen.contains("--port"), + "expected a bare `{command}` with no engine or port flag:\n{screen}" + ); +} + +#[then("the serve wizard approval closes and the wizard stays open")] +async fn serve_wizard_approval_closes_wizard_stays(world: &mut E2eWorld) { + // Waiting for the approval to clear between two Escapes keeps them from + // coalescing into one in the same PTY read, and shows the first one closed + // only the approval. + let tui = session(world); + tui.wait_until_gone( + "Port: automatic; endpoint shown after launch", + default_timeout(), + ) + .await + .unwrap_or_else(|e| panic!("the serve approval did not close: {e}")); + let screen = tui.screen_text(); + assert!( + screen.contains("Serve a model"), + "Escape on the approval closed the whole wizard:\n{screen}" + ); +} + +#[then(expr = "the serve wizard refuses to launch and says {string}")] +async fn serve_wizard_refuses_to_launch(world: &mut E2eWorld, message: String) { + let tui = session(world); + tui.wait_for_screen(&message, default_timeout()) + .await + .unwrap_or_else(|e| panic!("the serve wizard did not refuse the launch: {e}")); + let screen = tui.screen_text(); + assert!( + screen.contains("Serve a model") && !screen.contains("Port: automatic;"), + "a refused launch must leave the form open and stage nothing:\n{screen}" + ); +} + +#[then("the serve wizard is closed")] +async fn serve_wizard_closed(world: &mut E2eWorld) { + session(world) + .wait_until_gone("Serve a model", default_timeout()) + .await + .unwrap_or_else(|e| panic!("the serve wizard is still open after Escape: {e}")); +} + #[when("the user opens onboarding setup")] async fn open_onboarding_setup(world: &mut E2eWorld) { // `n` opens the onboarding wizard from the Observe tab diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index e8dc2b028..22e4d809e 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -9,7 +9,7 @@ use std::time::{Duration, Instant}; use cucumber::{given, then, when}; use crate::E2eWorld; -use e2e_cucumber::mock_server::{MockServer, ServiceRecordOptions}; +use e2e_cucumber::mock_server::{MockServer, ServiceRecordOptions, write_service_record_with}; use e2e_cucumber::serve_log::{ ServeAttempt, archive_service_log, serve_attempt_report, service_log_tail, }; @@ -394,6 +394,55 @@ async fn setup_mock_custom_port(world: &mut E2eWorld) { world.mock = Some(mock); } +#[given("the default serve port is occupied")] +async fn occupy_default_serve_port(world: &mut E2eWorld) { + let _ = ensure_serve_port_free().await; + // A stale listener from a prior scenario (not just an unrelated process) + // is the realistic way this probe finds the next port taken; clear it the + // same way `ensure_serve_port_free` clears the default port instead of + // panicking the whole run over leftover state. + kill_listeners_on_port(SERVE_PORT + 1); + let next = std::net::TcpListener::bind(("127.0.0.1", SERVE_PORT + 1)) + .expect("the next automatic port must be free before this scenario starts"); + drop(next); + world.automatic_port_guard = Some( + std::net::TcpListener::bind(("127.0.0.1", SERVE_PORT)) + .expect("failed to occupy the default serve port"), + ); +} + +/// Plant a live managed-service record on the default serve port, backed by a +/// real mock listener on that exact port. +/// +/// `ServiceRecordOptions::default()`'s `supervisor_pid: 0` means the CLI's +/// liveness refresh finds no tracked pid to check; without a real listener +/// behind the record, the endpoint probe would then come back `Unreachable` +/// and the record gets demoted to `"stopped"` *before* the explicit-port +/// collision check ever sees it (`refresh_managed_service_runtime_liveness` +/// in `apps/rocm/src/managed_service_status.rs`). Backing the record with a `MockServer` on +/// `SERVE_PORT` makes `/v1/models` list `other-model`, so the probe reports +/// `Listing`/`Serving` and the record is left `"ready"` instead — the +/// scenario needs the collision detected, so the port has to be reserved for +/// real, not just on paper. `supervisor_pid: 0` still keeps this safe for the +/// World's teardown, which runs `rocm services stop --yes` on every +/// record left in the isolated tree: a real PID here would point that stop at +/// the test process itself (see `plant_unauthenticated_lemonade_service`'s +/// doc comment for the same hazard). +#[given("another managed service already reserves the default serve port")] +async fn reserve_default_serve_port(world: &mut E2eWorld) { + ensure_serve_port_free().await; + let root = world.isolated_root.as_ref().expect("no isolated root"); + let services = root.path().join("data").join("services"); + let mock = MockServer::start_on_port("other-model", SERVE_PORT).await; + write_service_record_with( + &services, + "other-model", + SERVE_PORT, + ServiceRecordOptions::default(), + ); + world.mock = Some(mock); +} + /// The (model, engine, ready-substring) this host should serve for an /// engine-agnostic "serve a real model" precondition. /// @@ -933,6 +982,17 @@ async fn user_serves_with_failing_lemonade_preparation(world: &mut E2eWorld) { world.cli_rc = Some(rc); } +#[when("the user serves a model with automatic port selection")] +async fn user_serves_with_automatic_port(world: &mut E2eWorld) { + let (model, engine, _) = host_serve_target(); + let (stdout, stderr, rc) = + crate::run_rocm(world, &["serve", model, "--engine", engine, "--managed"]); + world.automatic_port_guard.take(); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + #[when("the user sends a chat completion request")] async fn user_sends_completion(world: &mut E2eWorld) { crate::send_chat(world).await; @@ -1103,8 +1163,78 @@ async fn user_serves_public_bind_no_optin(world: &mut E2eWorld) { world.cli_rc = Some(rc); } +/// `localhost` is loopback (`is_public_bind_host` returns false, so no +/// `--allow-public-bind` opt-in is required) but is not the canonical +/// `127.0.0.1` `validate_port_request` requires for automatic port selection. +/// This isolates the host-vs-port-explicitness check from the separate +/// public-bind-opt-in check `networking-03` already covers. +#[when("the user serves a model bound to a non-canonical host without an explicit port")] +async fn user_serves_non_canonical_host_no_port(world: &mut E2eWorld) { + let (stdout, stderr, rc) = + crate::run_rocm(world, &["serve", "some-model", "--host", "localhost"]); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +/// `--port 0` is rejected by `serve_port::parse_serve_port`'s own range check, +/// which also covers 65536 and non-numeric values via the same message. +#[when("the user serves a model with an out-of-range port")] +async fn user_serves_invalid_port(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm(world, &["serve", "some-model", "--port", "0"]); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[when("the user serves a model explicitly on the reserved port")] +async fn user_serves_explicit_reserved_port(world: &mut E2eWorld) { + let (model, engine, _) = host_serve_target(); + let port = SERVE_PORT.to_string(); + let (stdout, stderr, rc) = crate::run_rocm( + world, + &[ + "serve", + model, + "--engine", + engine, + "--managed", + "--port", + &port, + ], + ); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +/// Same CLI invocation as [`user_serves_explicit_reserved_port`]; the two +/// scenarios differ only in what is holding the port (a plain listener here, +/// vs. a live managed-service record there), which `resolve_serve_port` +/// routes to two different refusal messages. +#[when("the user serves a model explicitly on the occupied port")] +async fn user_serves_explicit_occupied_port(world: &mut E2eWorld) { + let (model, engine, _) = host_serve_target(); + let port = SERVE_PORT.to_string(); + let (stdout, stderr, rc) = crate::run_rocm( + world, + &[ + "serve", + model, + "--engine", + engine, + "--managed", + "--port", + &port, + ], + ); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + /// Serve naming both a runtime and an environment. These are mutually exclusive -/// selectors, rejected during argument parsing before any engine or GPU work — so +/// selectors, rejected during argument parsing before any engine or GPU work, so /// the refusal is observable with no GPU (mock lane). #[when("the user serves a model selecting both a runtime and an environment")] async fn user_serves_runtime_and_env(world: &mut E2eWorld) { @@ -1196,6 +1326,45 @@ async fn assert_public_bind_message(world: &mut E2eWorld) { ); } +#[then("the user is told to pass an explicit port for that host")] +async fn assert_custom_host_needs_port_message(world: &mut E2eWorld) { + let output = serve_output(world); + assert!( + output.contains("needs an explicit `--port"), + "expected a message asking for an explicit --port, got:\n{output}" + ); +} + +#[then("the user is told the port must be between 1 and 65535")] +async fn assert_invalid_port_message(world: &mut E2eWorld) { + let output = serve_output(world); + assert!( + output.contains("port must be a number between 1 and 65535"), + "expected the port range message, got:\n{output}" + ); +} + +#[then("the user is told to stop the service holding that port")] +async fn assert_port_reserved_message(world: &mut E2eWorld) { + let output = serve_output(world); + assert!( + output.contains("is already reserved by managed service e2e-mock") + && output.contains("rocm services stop e2e-mock --yes"), + "expected the port-reserved message naming e2e-mock, got:\n{output}" + ); +} + +#[then("the user is told the port is already in use by another process")] +async fn assert_port_in_use_message(world: &mut E2eWorld) { + let output = serve_output(world); + assert!( + output.contains(&format!( + "127.0.0.1:{SERVE_PORT} is already in use by another process" + )) && output.contains("Choose a free `--port`"), + "expected the port-in-use message, got:\n{output}" + ); +} + #[then("the user is told the two selectors cannot be combined")] async fn assert_selector_conflict_message(world: &mut E2eWorld) { let output = serve_output(world); @@ -1489,6 +1658,36 @@ async fn assert_endpoint_port(world: &mut E2eWorld) { ); } +#[then("the service uses the next automatic port")] +async fn assert_next_automatic_port(world: &mut E2eWorld) { + let stdout = world.cli_output.as_deref().unwrap_or(""); + let stderr = world.cli_stderr.as_deref().unwrap_or(""); + let rc = world.cli_rc.expect("no serve command was run"); + assert!( + rc == 0, + "{}", + e2e_cucumber::cli_failure_report( + &["serve", "", "--engine", "", "--managed"], + rc, + stdout, + stderr, + ) + ); + let expected = format!("endpoint: http://127.0.0.1:{}/v1", SERVE_PORT + 1); + let disclosure_line = stdout + .lines() + .position(|line| line.trim() == "Port: automatic; endpoint shown after launch"); + let endpoint_line = stdout.lines().position(|line| line.trim() == expected); + assert!( + endpoint_line.is_some(), + "expected `{expected}` in real serve output:\n{stdout}" + ); + assert!( + disclosure_line.is_some_and(|d| d < endpoint_line.unwrap()), + "the automatic-port disclosure must print before the resolved endpoint:\n{stdout}" + ); +} + /// Extract the engine name from a serve plan's `engine: ` line. fn selected_engine(output: &str) -> &str { let Some(engine) = output @@ -1732,6 +1931,118 @@ async fn user_serves_requiring_api_key(world: &mut E2eWorld) { world.cli_rc = Some(rc); } +#[when("the user serves that same model again on an explicit port other than the running one")] +async fn user_serves_other_explicit_port(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm( + world, + &[ + "serve", + REUSE_REFUSAL_MODEL, + "--engine", + REUSE_REFUSAL_ENGINE, + "--managed", + "--port", + "11500", + ], + ); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[then("the CLI refuses and names the port the running service uses")] +async fn assert_refused_naming_running_port(world: &mut E2eWorld) { + let rc = world.cli_rc.expect("no serve rc recorded"); + let output = serve_output(world); + assert!( + rc != 0, + "an explicit port the running service does not use must be refused:\n{output}" + ); + assert!( + output.contains(REUSE_REFUSAL_SERVICE_ID) + && output.contains("on port 9") + && output.contains("port 11500 would not be used"), + "the refusal must name the running service and both ports:\n{output}" + ); +} + +#[then("the CLI names the exact command to stop the existing service")] +async fn assert_names_stop_command(world: &mut E2eWorld) { + let output = serve_output(world); + assert!( + output.contains(&format!( + "rocm services stop {REUSE_REFUSAL_SERVICE_ID} --yes" + )), + "the refusal must name the way out, directly runnable with --yes: {output}" + ); +} + +/// A second record, `e2e-stopped`, on the default serve port: the service the +/// restart scenario tries to bring back. Cloned from the `e2e-mock` record +/// `reserve_default_serve_port` planted, so its shape is the one the CLI reads. +#[given("a stopped managed service is recorded on the default serve port")] +async fn plant_stopped_service_on_default_port(world: &mut E2eWorld) { + let services = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("data") + .join("services"); + let mut record: serde_json::Value = serde_json::from_slice( + &std::fs::read(services.join("e2e-mock.json")).expect("read e2e-mock record"), + ) + .expect("parse e2e-mock record"); + record["service_id"] = "e2e-stopped".into(); + record["status"] = "stopped".into(); + record["manifest_path"] = services.join("e2e-stopped.json").to_string_lossy().into(); + record["log_path"] = services.join("e2e-stopped.log").to_string_lossy().into(); + record["engine_state_path"] = services + .join("e2e-stopped.state.json") + .to_string_lossy() + .into(); + std::fs::write( + services.join("e2e-stopped.json"), + serde_json::to_vec_pretty(&record).expect("serialize record"), + ) + .expect("write stopped record"); +} + +#[when("the user restarts the stopped service")] +async fn user_restarts_stopped_service(world: &mut E2eWorld) { + let (stdout, stderr, rc) = + crate::run_rocm(world, &["services", "restart", "e2e-stopped", "--yes"]); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[then("the restart is refused naming the service that holds the port")] +async fn assert_restart_refused_naming_holder(world: &mut E2eWorld) { + let rc = world.cli_rc.expect("no restart rc recorded"); + let output = serve_output(world); + assert!( + rc != 0, + "the restart must be refused, but exited 0:\n{output}" + ); + assert!( + output.contains("e2e-mock") && output.contains("recorded on that port"), + "the refusal must name the service holding the port:\n{output}" + ); + // Nothing was spawned: the stopped record is still stopped. + let root = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path(); + let record: serde_json::Value = serde_json::from_slice( + &std::fs::read(root.join("data").join("services").join("e2e-stopped.json")) + .expect("read stopped record"), + ) + .expect("parse stopped record"); + assert_eq!(record["status"], "stopped", "{record}"); +} + #[then("the CLI refuses and names the exact command to stop the existing service")] async fn assert_refused_with_stop_hint(world: &mut E2eWorld) { let rc = world.cli_rc.expect("no serve rc recorded");