From ac3fc481cd76854050c76b40071d1180644a86d6 Mon Sep 17 00:00:00 2001 From: mjamiv <142179942+mjamiv@users.noreply.github.com> Date: Wed, 15 Apr 2026 11:02:35 -0500 Subject: [PATCH 001/142] fix(core): exclude vm-dev tag from git describe version glob (#843) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `git describe --tags --long --match "v*"` matches the `vm-dev` tag alongside release tags. When `vm-dev` sits on or past the latest release tag in the commit graph (currently the case on main), git picks it, and the resulting `vm-dev-N-gSHA` string strips the leading `v` down to `m-dev-N-gSHA`. With `commits == 0` that's returned verbatim, producing a binary that reports itself as `m-dev`: $ openshell --version openshell m-dev Confirmed on v0.0.28 (#832) and reproduced on v0.0.29 as well. Restrict the glob to numeric release tags (`v[0-9]*`). All release tags follow `v\d+\.\d+\.\d+`, so this loses no valid version — it only filters out `vm-dev`, `vm-prod`, and any similar non-release tags that share the `v` prefix. Verified locally on this repo's current HEAD: # before $ git describe --tags --long --match 'v*' vm-dev-0-g355d845 # after $ git describe --tags --long --match 'v[0-9]*' v0.0.29-2-g355d845 Closes #832 Signed-off-by: mjamiv --- crates/openshell-core/build.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/openshell-core/build.rs b/crates/openshell-core/build.rs index c89a03483c..e57de44654 100644 --- a/crates/openshell-core/build.rs +++ b/crates/openshell-core/build.rs @@ -63,8 +63,14 @@ fn main() -> Result<(), Box> { /// /// Returns `None` when git is unavailable or the repo has no matching tags. fn git_version() -> Option { + // Match numeric release tags only (e.g. `v0.0.29`). The bare glob `v*` + // also matches non-release tags like `vm-dev` or `vm-prod`; when one of + // those lands on the same commit as a release tag, `git describe` picks + // it and the resulting version string collapses to `m-dev` after the + // leading `v` is stripped below. Requiring a digit after `v` excludes + // those development tags without losing any release tag. let output = std::process::Command::new("git") - .args(["describe", "--tags", "--long", "--match", "v*"]) + .args(["describe", "--tags", "--long", "--match", "v[0-9]*"]) .output() .ok()?; From 25d2530b3c581db307caf0ef0607f8bd73938fcb Mon Sep 17 00:00:00 2001 From: "John T. Myers" <9696606+johntmyers@users.noreply.github.com> Date: Wed, 15 Apr 2026 15:19:18 -0700 Subject: [PATCH 002/142] fix(inference): allowlist routed request headers (#826) Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- architecture/inference-routing.md | 13 +- architecture/sandbox.md | 4 +- crates/openshell-core/src/inference.rs | 64 ++++++- crates/openshell-router/src/backend.rs | 161 ++++++++++++++++-- crates/openshell-router/src/config.rs | 41 +++-- crates/openshell-router/src/mock.rs | 1 + .../tests/backend_integration.rs | 50 ++++++ crates/openshell-sandbox/src/lib.rs | 8 +- crates/openshell-sandbox/src/proxy.rs | 147 ++++++++++------ .../tests/system_inference.rs | 6 + crates/openshell-server/src/inference.rs | 5 + docs/inference/about.mdx | 5 +- 12 files changed, 412 insertions(+), 93 deletions(-) diff --git a/architecture/inference-routing.md b/architecture/inference-routing.md index e2fd671782..c0c42f4f6a 100644 --- a/architecture/inference-routing.md +++ b/architecture/inference-routing.md @@ -171,16 +171,17 @@ Files: `prepare_backend_request()` (shared by both buffered and streaming paths) rewrites outgoing requests: 1. **Auth injection**: Uses the route's `AuthHeader` -- either `Authorization: Bearer ` or a custom header (e.g. `x-api-key: ` for Anthropic). -2. **Header stripping**: Removes `authorization`, `x-api-key`, `host`, and any header names that will be set from route defaults. -3. **Default headers**: Applies route-level default headers (e.g. `anthropic-version: 2023-06-01`) unless the client already sent them. -4. **Model rewrite**: Parses the request body as JSON and replaces the `model` field with the route's configured model. Non-JSON bodies are forwarded unchanged. -5. **URL construction**: `build_backend_url()` appends the request path to the route endpoint. If the endpoint already ends with `/v1` and the request path starts with `/v1/`, the duplicate prefix is deduplicated. +2. **Header allowlist**: Keeps only explicitly approved request headers: common inference headers (`content-type`, `accept`, `accept-encoding`, `user-agent`), route-specific passthrough headers (for example `openai-organization`, `x-model-id`, `anthropic-version`, `anthropic-beta`), and any route default header names. +3. **Header stripping**: Removes `authorization`, `x-api-key`, `host`, `content-length`, hop-by-hop headers, and any non-allowlisted request headers. +4. **Default headers**: Applies route-level default headers (e.g. `anthropic-version: 2023-06-01`) unless the client already sent them. +5. **Model rewrite**: Parses the request body as JSON and replaces the `model` field with the route's configured model. Non-JSON bodies are forwarded unchanged. +6. **URL construction**: `build_backend_url()` appends the request path to the route endpoint. If the endpoint already ends with `/v1` and the request path starts with `/v1/`, the duplicate prefix is deduplicated. ### Header sanitization -Before forwarding inference requests, the proxy strips sensitive and hop-by-hop headers from both requests and responses: +Before forwarding inference requests, the router enforces a route-aware request allowlist and strips sensitive/framing headers. Response sanitization remains framing-only: -- **Request**: `authorization`, `x-api-key`, `host`, `content-length`, and hop-by-hop headers (`connection`, `keep-alive`, `proxy-authenticate`, `proxy-authorization`, `proxy-connection`, `te`, `trailer`, `transfer-encoding`, `upgrade`). +- **Request**: forwards only common inference headers plus route-specific passthrough headers and route default header names. Always strips `authorization`, `x-api-key`, `host`, `content-length`, unknown headers such as `cookie`, and hop-by-hop headers (`connection`, `keep-alive`, `proxy-authenticate`, `proxy-authorization`, `proxy-connection`, `te`, `trailer`, `transfer-encoding`, `upgrade`). - **Response**: `content-length` and hop-by-hop headers. ### Response streaming diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 096a1bb608..c6e715f259 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -839,9 +839,9 @@ The interception steps: Pattern matching strips query strings. Exact path comparison is used for most patterns; the `/v1/models/*` pattern matches `/v1/models` itself or any path under `/v1/models/` (e.g., `/v1/models/gpt-4.1`). -4. **Header sanitization**: For matched inference requests, the proxy strips credential headers (`Authorization`, `x-api-key`) and framing/hop-by-hop headers (`host`, `content-length`, `transfer-encoding`, `connection`, etc.). The router rebuilds correct framing for the forwarded body. +4. **Header sanitization**: For matched inference requests, the proxy passes the parsed headers to the router. The router applies a route-aware allowlist before forwarding: common inference headers (`content-type`, `accept`, `accept-encoding`, `user-agent`), provider-specific passthrough headers (for example `openai-organization`, `x-model-id`, `anthropic-version`, `anthropic-beta`), and any route default header names. It always strips client-supplied credential headers (`Authorization`, `x-api-key`) and framing/hop-by-hop headers (`host`, `content-length`, `transfer-encoding`, `connection`, etc.). The router rebuilds correct framing for the forwarded body. -5. **Local routing**: Matched requests are executed by calling `Router::proxy_with_candidates_streaming()`, passing the detected protocol, HTTP method, path, sanitized headers, body, and the cached `ResolvedRoute` list from `InferenceContext`. The router selects the first route whose `protocols` list contains the source protocol (see [Inference Routing -- Response streaming](inference-routing.md#response-streaming) for details). When forwarding to the backend, the router rewrites the request: the route's `api_key` replaces the `Authorization` header, the `Host` header is set to the backend endpoint, and the `"model"` field in the JSON request body is replaced with the route's configured `model` value. If the request body is not valid JSON or does not contain a `"model"` key, the body is forwarded unchanged. +5. **Local routing**: Matched requests are executed by calling `Router::proxy_with_candidates_streaming()`, passing the detected protocol, HTTP method, path, original parsed headers, body, and the cached `ResolvedRoute` list from `InferenceContext`. The router selects the first route whose `protocols` list contains the source protocol (see [Inference Routing -- Response streaming](inference-routing.md#response-streaming) for details). When forwarding to the backend, the router rewrites the request: the route's `api_key` replaces the client auth header, the `Host` header is set to the backend endpoint, only allowlisted request headers survive, and the `"model"` field in the JSON request body is replaced with the route's configured `model` value. If the request body is not valid JSON or does not contain a `"model"` key, the body is forwarded unchanged. 6. **Response handling (streaming)**: - On success: response headers are sent back to the client immediately as an HTTP/1.1 response with `Transfer-Encoding: chunked`, using `format_http_response_header()`. Framing/hop-by-hop headers are stripped from the upstream response. Body chunks are then forwarded incrementally as they arrive from the backend via `StreamingProxyResponse::next_chunk()`, each wrapped in HTTP chunked encoding by `format_chunk()`. The stream is terminated with a `0\r\n\r\n` chunk terminator. This ensures time-to-first-byte reflects the backend's first token latency rather than the full generation time. diff --git a/crates/openshell-core/src/inference.rs b/crates/openshell-core/src/inference.rs index a06c427f8b..d2581f7eb0 100644 --- a/crates/openshell-core/src/inference.rs +++ b/crates/openshell-core/src/inference.rs @@ -28,7 +28,8 @@ pub enum AuthHeader { /// /// This is the single source of truth for provider-specific inference knowledge: /// default endpoint, supported protocols, credential key lookup order, auth -/// header style, and default headers. +/// header style, default headers, and allowed client-supplied passthrough +/// headers. /// /// This is separate from [`openshell_providers::ProviderPlugin`] which handles /// credential *discovery* (scanning env vars). `InferenceProviderProfile` handles @@ -45,6 +46,10 @@ pub struct InferenceProviderProfile { pub auth: AuthHeader, /// Default headers injected on every outgoing request. pub default_headers: &'static [(&'static str, &'static str)], + /// Client-supplied headers that may be forwarded to the upstream backend. + /// + /// Header names must be lowercase and must not include auth headers. + pub passthrough_headers: &'static [&'static str], } const OPENAI_PROTOCOLS: &[&str] = &[ @@ -64,6 +69,7 @@ static OPENAI_PROFILE: InferenceProviderProfile = InferenceProviderProfile { base_url_config_keys: &["OPENAI_BASE_URL"], auth: AuthHeader::Bearer, default_headers: &[], + passthrough_headers: &["openai-organization", "x-model-id"], }; static ANTHROPIC_PROFILE: InferenceProviderProfile = InferenceProviderProfile { @@ -74,6 +80,7 @@ static ANTHROPIC_PROFILE: InferenceProviderProfile = InferenceProviderProfile { base_url_config_keys: &["ANTHROPIC_BASE_URL"], auth: AuthHeader::Custom("x-api-key"), default_headers: &[("anthropic-version", "2023-06-01")], + passthrough_headers: &["anthropic-version", "anthropic-beta"], }; static NVIDIA_PROFILE: InferenceProviderProfile = InferenceProviderProfile { @@ -84,6 +91,7 @@ static NVIDIA_PROFILE: InferenceProviderProfile = InferenceProviderProfile { base_url_config_keys: &["NVIDIA_BASE_URL"], auth: AuthHeader::Bearer, default_headers: &[], + passthrough_headers: &["x-model-id"], }; /// Look up the inference provider profile for a given provider type. @@ -105,6 +113,17 @@ pub fn profile_for(provider_type: &str) -> Option<&'static InferenceProviderProf /// need the auth/header information (e.g. the sandbox bundle-to-route /// conversion). pub fn auth_for_provider_type(provider_type: &str) -> (AuthHeader, Vec<(String, String)>) { + let (auth, headers, _) = route_headers_for_provider_type(provider_type); + (auth, headers) +} + +/// Derive routing header policy for a provider type string. +/// +/// Returns the auth injection mode, route-level default headers, and the +/// allowed client-supplied passthrough headers for `inference.local`. +pub fn route_headers_for_provider_type( + provider_type: &str, +) -> (AuthHeader, Vec<(String, String)>, Vec) { match profile_for(provider_type) { Some(profile) => { let headers = profile @@ -112,9 +131,14 @@ pub fn auth_for_provider_type(provider_type: &str) -> (AuthHeader, Vec<(String, .iter() .map(|(k, v)| ((*k).to_string(), (*v).to_string())) .collect(); - (profile.auth.clone(), headers) + let passthrough_headers = profile + .passthrough_headers + .iter() + .map(|name| (*name).to_string()) + .collect(); + (profile.auth.clone(), headers, passthrough_headers) } - None => (AuthHeader::Bearer, Vec::new()), + None => (AuthHeader::Bearer, Vec::new(), Vec::new()), } } @@ -193,6 +217,32 @@ mod tests { assert!(headers.iter().any(|(k, _)| k == "anthropic-version")); } + #[test] + fn route_headers_for_openai_include_passthrough_headers() { + let (_, _, passthrough_headers) = route_headers_for_provider_type("openai"); + assert!( + passthrough_headers + .iter() + .any(|name| name == "openai-organization") + ); + assert!(passthrough_headers.iter().any(|name| name == "x-model-id")); + } + + #[test] + fn route_headers_for_anthropic_include_passthrough_headers() { + let (_, _, passthrough_headers) = route_headers_for_provider_type("anthropic"); + assert!( + passthrough_headers + .iter() + .any(|name| name == "anthropic-version") + ); + assert!( + passthrough_headers + .iter() + .any(|name| name == "anthropic-beta") + ); + } + #[test] fn auth_for_openai_uses_bearer() { let (auth, headers) = auth_for_provider_type("openai"); @@ -206,4 +256,12 @@ mod tests { assert_eq!(auth, AuthHeader::Bearer); assert!(headers.is_empty()); } + + #[test] + fn route_headers_for_unknown_are_empty() { + let (auth, headers, passthrough_headers) = route_headers_for_provider_type("unknown"); + assert_eq!(auth, AuthHeader::Bearer); + assert!(headers.is_empty()); + assert!(passthrough_headers.is_empty()); + } } diff --git a/crates/openshell-router/src/backend.rs b/crates/openshell-router/src/backend.rs index 8dbae65020..fbca70ae18 100644 --- a/crates/openshell-router/src/backend.rs +++ b/crates/openshell-router/src/backend.rs @@ -4,6 +4,7 @@ use crate::RouterError; use crate::config::{AuthHeader, ResolvedRoute}; use crate::mock; +use std::collections::HashSet; #[derive(Debug, Clone, PartialEq, Eq)] pub struct ValidatedEndpoint { @@ -62,6 +63,9 @@ enum StreamingBody { Buffered(Option), } +const COMMON_INFERENCE_REQUEST_HEADERS: [&str; 4] = + ["content-type", "accept", "accept-encoding", "user-agent"]; + impl StreamingProxyResponse { /// Create from a fully-buffered [`ProxyResponse`] (for mock routes). pub fn from_buffered(resp: ProxyResponse) -> Self { @@ -83,7 +87,64 @@ impl StreamingProxyResponse { } } -/// Build an HTTP request to the backend configured in `route`. +fn sanitize_request_headers( + route: &ResolvedRoute, + headers: &[(String, String)], +) -> Vec<(String, String)> { + let mut allowed = HashSet::new(); + allowed.extend( + COMMON_INFERENCE_REQUEST_HEADERS + .iter() + .map(|name| (*name).to_string()), + ); + allowed.extend( + route + .passthrough_headers + .iter() + .map(|name| name.to_ascii_lowercase()), + ); + allowed.extend( + route + .default_headers + .iter() + .map(|(name, _)| name.to_ascii_lowercase()), + ); + + headers + .iter() + .filter_map(|(name, value)| { + let name_lc = name.to_ascii_lowercase(); + if should_strip_request_header(&name_lc) || !allowed.contains(&name_lc) { + return None; + } + Some((name.clone(), value.clone())) + }) + .collect() +} + +fn should_strip_request_header(name: &str) -> bool { + matches!( + name, + "authorization" | "x-api-key" | "host" | "content-length" + ) || is_hop_by_hop_header(name) +} + +fn is_hop_by_hop_header(name: &str) -> bool { + matches!( + name, + "connection" + | "keep-alive" + | "proxy-authenticate" + | "proxy-authorization" + | "proxy-connection" + | "te" + | "trailer" + | "transfer-encoding" + | "upgrade" + ) +} + +/// Build and send an HTTP request to the backend configured in `route`. /// /// Returns the prepared [`reqwest::RequestBuilder`] with auth, headers, model /// rewrite, and body applied. The caller decides whether to apply a total @@ -97,6 +158,7 @@ fn prepare_backend_request( body: bytes::Bytes, ) -> Result<(reqwest::RequestBuilder, String), RouterError> { let url = build_backend_url(&route.endpoint, path); + let headers = sanitize_request_headers(route, &headers); let reqwest_method: reqwest::Method = method .parse() @@ -113,17 +175,7 @@ fn prepare_backend_request( builder = builder.header(*header_name, &route.api_key); } } - - // Strip auth and host headers — auth is re-injected above from the route - // config, and host must match the upstream. - let strip_headers: [&str; 3] = ["authorization", "x-api-key", "host"]; - - // Forward non-sensitive headers. - for (name, value) in headers { - let name_lc = name.to_ascii_lowercase(); - if strip_headers.contains(&name_lc.as_str()) { - continue; - } + for (name, value) in &headers { builder = builder.header(name.as_str(), value.as_str()); } @@ -510,10 +562,95 @@ mod tests { protocols: protocols.iter().map(|p| (*p).to_string()).collect(), auth, default_headers: vec![("anthropic-version".to_string(), "2023-06-01".to_string())], + passthrough_headers: vec![ + "anthropic-version".to_string(), + "anthropic-beta".to_string(), + ], timeout: crate::config::DEFAULT_ROUTE_TIMEOUT, } } + #[test] + fn sanitize_request_headers_drops_unknown_sensitive_headers() { + let route = ResolvedRoute { + name: "inference.local".to_string(), + endpoint: "https://api.example.com/v1".to_string(), + model: "test-model".to_string(), + api_key: "sk-test".to_string(), + protocols: vec!["openai_chat_completions".to_string()], + auth: AuthHeader::Bearer, + default_headers: Vec::new(), + passthrough_headers: vec!["openai-organization".to_string()], + timeout: crate::config::DEFAULT_ROUTE_TIMEOUT, + }; + + let kept = super::sanitize_request_headers( + &route, + &[ + ("content-type".to_string(), "application/json".to_string()), + ("authorization".to_string(), "Bearer client".to_string()), + ("cookie".to_string(), "session=1".to_string()), + ("x-amz-security-token".to_string(), "token".to_string()), + ("openai-organization".to_string(), "org_123".to_string()), + ], + ); + + assert!( + kept.iter() + .any(|(name, _)| name.eq_ignore_ascii_case("content-type")) + ); + assert!( + kept.iter() + .any(|(name, _)| name.eq_ignore_ascii_case("openai-organization")) + ); + assert!( + kept.iter() + .all(|(name, _)| !name.eq_ignore_ascii_case("authorization")) + ); + assert!( + kept.iter() + .all(|(name, _)| !name.eq_ignore_ascii_case("cookie")) + ); + assert!( + kept.iter() + .all(|(name, _)| !name.eq_ignore_ascii_case("x-amz-security-token")) + ); + } + + #[test] + fn sanitize_request_headers_preserves_allowed_provider_headers() { + let route = test_route( + "https://api.anthropic.com/v1", + &["anthropic_messages"], + AuthHeader::Custom("x-api-key"), + ); + + let kept = super::sanitize_request_headers( + &route, + &[ + ("anthropic-version".to_string(), "2024-10-22".to_string()), + ( + "anthropic-beta".to_string(), + "tool-use-2024-10-22".to_string(), + ), + ("x-api-key".to_string(), "client-key".to_string()), + ], + ); + + assert!(kept.iter().any( + |(name, value)| name.eq_ignore_ascii_case("anthropic-version") && value == "2024-10-22" + )); + assert!( + kept.iter() + .any(|(name, value)| name.eq_ignore_ascii_case("anthropic-beta") + && value == "tool-use-2024-10-22") + ); + assert!( + kept.iter() + .all(|(name, _)| !name.eq_ignore_ascii_case("x-api-key")) + ); + } + #[tokio::test] async fn verify_backend_endpoint_uses_route_auth_and_shape() { let mock_server = MockServer::start().await; diff --git a/crates/openshell-router/src/config.rs b/crates/openshell-router/src/config.rs index b531e091d9..660509d9e5 100644 --- a/crates/openshell-router/src/config.rs +++ b/crates/openshell-router/src/config.rs @@ -34,7 +34,7 @@ pub struct RouteConfig { /// A fully-resolved route ready for the router to forward requests. /// /// The router is provider-agnostic — all provider-specific decisions -/// (auth header style, default headers, base URL) are made by the +/// (auth header style, default headers, passthrough headers, base URL) are made by the /// caller during resolution. #[derive(Clone)] pub struct ResolvedRoute { @@ -48,6 +48,8 @@ pub struct ResolvedRoute { pub auth: AuthHeader, /// Extra headers injected on every request (e.g. `anthropic-version`). pub default_headers: Vec<(String, String)>, + /// Client-supplied headers that may be forwarded to the upstream backend. + pub passthrough_headers: Vec, /// Per-request timeout for proxied inference calls. pub timeout: Duration, } @@ -62,6 +64,7 @@ impl std::fmt::Debug for ResolvedRoute { .field("protocols", &self.protocols) .field("auth", &self.auth) .field("default_headers", &self.default_headers) + .field("passthrough_headers", &self.passthrough_headers) .field("timeout", &self.timeout) .finish() } @@ -125,7 +128,8 @@ impl RouteConfig { ))); } - let (auth, default_headers) = auth_from_provider_type(self.provider_type.as_deref()); + let (auth, default_headers, passthrough_headers) = + route_headers_from_provider_type(self.provider_type.as_deref()); Ok(ResolvedRoute { name: self.name.clone(), @@ -135,17 +139,21 @@ impl RouteConfig { protocols, auth, default_headers, + passthrough_headers, timeout: DEFAULT_ROUTE_TIMEOUT, }) } } -/// Derive auth header style and default headers from a provider type string. +/// Derive auth header style, default headers, and passthrough headers from a +/// provider type string. /// -/// Delegates to [`openshell_core::inference::auth_for_provider_type`] which -/// uses the centralized `InferenceProviderProfile` registry. -fn auth_from_provider_type(provider_type: Option<&str>) -> (AuthHeader, Vec<(String, String)>) { - openshell_core::inference::auth_for_provider_type(provider_type.unwrap_or("")) +/// Delegates to [`openshell_core::inference::route_headers_for_provider_type`] +/// which uses the centralized `InferenceProviderProfile` registry. +fn route_headers_from_provider_type( + provider_type: Option<&str>, +) -> (AuthHeader, Vec<(String, String)>, Vec) { + openshell_core::inference::route_headers_for_provider_type(provider_type.unwrap_or("")) } #[cfg(test)] @@ -263,6 +271,7 @@ routes: protocols: vec!["openai_chat_completions".to_string()], auth: AuthHeader::Bearer, default_headers: Vec::new(), + passthrough_headers: Vec::new(), timeout: DEFAULT_ROUTE_TIMEOUT, }; let debug_output = format!("{route:?}"); @@ -278,22 +287,34 @@ routes: #[test] fn auth_from_anthropic_provider_uses_custom_header() { - let (auth, headers) = auth_from_provider_type(Some("anthropic")); + let (auth, headers, passthrough_headers) = + route_headers_from_provider_type(Some("anthropic")); assert_eq!(auth, AuthHeader::Custom("x-api-key")); assert!(headers.iter().any(|(k, _)| k == "anthropic-version")); + assert!( + passthrough_headers + .iter() + .any(|name| name == "anthropic-beta") + ); } #[test] fn auth_from_openai_provider_uses_bearer() { - let (auth, headers) = auth_from_provider_type(Some("openai")); + let (auth, headers, passthrough_headers) = route_headers_from_provider_type(Some("openai")); assert_eq!(auth, AuthHeader::Bearer); assert!(headers.is_empty()); + assert!( + passthrough_headers + .iter() + .any(|name| name == "openai-organization") + ); } #[test] fn auth_from_none_defaults_to_bearer() { - let (auth, headers) = auth_from_provider_type(None); + let (auth, headers, passthrough_headers) = route_headers_from_provider_type(None); assert_eq!(auth, AuthHeader::Bearer); assert!(headers.is_empty()); + assert!(passthrough_headers.is_empty()); } } diff --git a/crates/openshell-router/src/mock.rs b/crates/openshell-router/src/mock.rs index a17ce486f0..66fc804144 100644 --- a/crates/openshell-router/src/mock.rs +++ b/crates/openshell-router/src/mock.rs @@ -131,6 +131,7 @@ mod tests { protocols: protocols.iter().map(ToString::to_string).collect(), auth: crate::config::AuthHeader::Bearer, default_headers: Vec::new(), + passthrough_headers: Vec::new(), timeout: crate::config::DEFAULT_ROUTE_TIMEOUT, } } diff --git a/crates/openshell-router/tests/backend_integration.rs b/crates/openshell-router/tests/backend_integration.rs index d9aecb0e37..6b21de94dd 100644 --- a/crates/openshell-router/tests/backend_integration.rs +++ b/crates/openshell-router/tests/backend_integration.rs @@ -15,6 +15,7 @@ fn mock_candidates(base_url: &str) -> Vec { protocols: vec!["openai_chat_completions".to_string()], auth: AuthHeader::Bearer, default_headers: Vec::new(), + passthrough_headers: vec!["openai-organization".to_string(), "x-model-id".to_string()], timeout: openshell_router::config::DEFAULT_ROUTE_TIMEOUT, }] } @@ -118,6 +119,7 @@ async fn proxy_no_compatible_route_returns_error() { protocols: vec!["anthropic_messages".to_string()], auth: AuthHeader::Custom("x-api-key"), default_headers: Vec::new(), + passthrough_headers: Vec::new(), timeout: openshell_router::config::DEFAULT_ROUTE_TIMEOUT, }]; @@ -169,6 +171,39 @@ async fn proxy_strips_auth_header() { assert_eq!(response.status, 200); } +#[tokio::test] +async fn proxy_forwards_openai_organization_header() { + let mock_server = MockServer::start().await; + + Mock::given(method("POST")) + .and(path("/v1/chat/completions")) + .and(bearer_token("test-api-key")) + .and(header("openai-organization", "org_123")) + .respond_with(ResponseTemplate::new(200).set_body_string("{}")) + .mount(&mock_server) + .await; + + let router = Router::new().unwrap(); + let candidates = mock_candidates(&mock_server.uri()); + + let response = router + .proxy_with_candidates( + "openai_chat_completions", + "POST", + "/v1/chat/completions", + vec![ + ("openai-organization".to_string(), "org_123".to_string()), + ("cookie".to_string(), "session=abc".to_string()), + ], + bytes::Bytes::new(), + &candidates, + ) + .await + .unwrap(); + + assert_eq!(response.status, 200); +} + #[tokio::test] async fn proxy_mock_route_returns_canned_response() { let router = Router::new().unwrap(); @@ -180,6 +215,7 @@ async fn proxy_mock_route_returns_canned_response() { protocols: vec!["openai_chat_completions".to_string()], auth: AuthHeader::Bearer, default_headers: Vec::new(), + passthrough_headers: Vec::new(), timeout: openshell_router::config::DEFAULT_ROUTE_TIMEOUT, }]; @@ -315,6 +351,10 @@ async fn proxy_uses_x_api_key_for_anthropic_route() { protocols: vec!["anthropic_messages".to_string()], auth: AuthHeader::Custom("x-api-key"), default_headers: vec![("anthropic-version".to_string(), "2023-06-01".to_string())], + passthrough_headers: vec![ + "anthropic-version".to_string(), + "anthropic-beta".to_string(), + ], timeout: openshell_router::config::DEFAULT_ROUTE_TIMEOUT, }]; @@ -374,6 +414,10 @@ async fn proxy_anthropic_does_not_send_bearer_auth() { protocols: vec!["anthropic_messages".to_string()], auth: AuthHeader::Custom("x-api-key"), default_headers: vec![("anthropic-version".to_string(), "2023-06-01".to_string())], + passthrough_headers: vec![ + "anthropic-version".to_string(), + "anthropic-beta".to_string(), + ], timeout: openshell_router::config::DEFAULT_ROUTE_TIMEOUT, }]; @@ -419,6 +463,10 @@ async fn proxy_forwards_client_anthropic_version_header() { protocols: vec!["anthropic_messages".to_string()], auth: AuthHeader::Custom("x-api-key"), default_headers: vec![("anthropic-version".to_string(), "2023-06-01".to_string())], + passthrough_headers: vec![ + "anthropic-version".to_string(), + "anthropic-beta".to_string(), + ], timeout: openshell_router::config::DEFAULT_ROUTE_TIMEOUT, }]; @@ -509,6 +557,7 @@ async fn streaming_proxy_completes_despite_exceeding_route_timeout() { protocols: vec!["openai_chat_completions".to_string()], auth: AuthHeader::Bearer, default_headers: Vec::new(), + passthrough_headers: Vec::new(), // Route timeout shorter than the backend delay — streaming must // NOT be constrained by this. timeout: Duration::from_secs(1), @@ -572,6 +621,7 @@ async fn buffered_proxy_enforces_route_timeout() { protocols: vec!["openai_chat_completions".to_string()], auth: AuthHeader::Bearer, default_headers: Vec::new(), + passthrough_headers: Vec::new(), timeout: Duration::from_secs(1), }]; diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index c2956b1e09..b81dd4a6c2 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -1099,8 +1099,8 @@ pub(crate) fn bundle_to_resolved_routes( .routes .iter() .map(|r| { - let (auth, default_headers) = - openshell_core::inference::auth_for_provider_type(&r.provider_type); + let (auth, default_headers, passthrough_headers) = + openshell_core::inference::route_headers_for_provider_type(&r.provider_type); let timeout = if r.timeout_secs == 0 { openshell_router::config::DEFAULT_ROUTE_TIMEOUT } else { @@ -1114,6 +1114,7 @@ pub(crate) fn bundle_to_resolved_routes( protocols: r.protocols.clone(), auth, default_headers, + passthrough_headers, timeout, } }) @@ -2296,6 +2297,7 @@ mod tests { protocols: vec!["openai_chat_completions".to_string()], auth: openshell_core::inference::AuthHeader::Bearer, default_headers: vec![], + passthrough_headers: vec![], timeout: openshell_router::config::DEFAULT_ROUTE_TIMEOUT, }, openshell_router::config::ResolvedRoute { @@ -2306,6 +2308,7 @@ mod tests { protocols: vec!["anthropic_messages".to_string()], auth: openshell_core::inference::AuthHeader::Custom("x-api-key"), default_headers: vec![], + passthrough_headers: vec![], timeout: openshell_router::config::DEFAULT_ROUTE_TIMEOUT, }, ]; @@ -2595,6 +2598,7 @@ filesystem_policy: auth: openshell_core::inference::AuthHeader::Bearer, protocols: vec!["openai_chat_completions".to_string()], default_headers: vec![], + passthrough_headers: vec![], timeout: openshell_router::config::DEFAULT_ROUTE_TIMEOUT, }]; diff --git a/crates/openshell-sandbox/src/proxy.rs b/crates/openshell-sandbox/src/proxy.rs index 1b87a0c7f2..6f85e848e2 100644 --- a/crates/openshell-sandbox/src/proxy.rs +++ b/crates/openshell-sandbox/src/proxy.rs @@ -1205,9 +1205,6 @@ async fn route_inference_request( ocsf_emit!(event); } - // Strip credential + framing/hop-by-hop headers. - let filtered_headers = sanitize_inference_request_headers(&request.headers); - let routes = ctx.routes.read().await; if routes.is_empty() { @@ -1231,7 +1228,7 @@ async fn route_inference_request( &pattern.protocol, &request.method, &normalized_path, - filtered_headers, + request.headers.clone(), bytes::Bytes::from(request.body.clone()), &routes, ) @@ -1393,14 +1390,6 @@ fn router_error_to_http(err: &openshell_router::RouterError) -> (u16, String) { } } -fn sanitize_inference_request_headers(headers: &[(String, String)]) -> Vec<(String, String)> { - headers - .iter() - .filter(|(name, _)| !should_strip_request_header(name)) - .cloned() - .collect() -} - fn sanitize_inference_response_headers(headers: Vec<(String, String)>) -> Vec<(String, String)> { headers .into_iter() @@ -1408,14 +1397,6 @@ fn sanitize_inference_response_headers(headers: Vec<(String, String)>) -> Vec<(S .collect() } -fn should_strip_request_header(name: &str) -> bool { - let name_lc = name.to_ascii_lowercase(); - matches!( - name_lc.as_str(), - "authorization" | "x-api-key" | "host" | "content-length" - ) || is_hop_by_hop_header(&name_lc) -} - fn should_strip_response_header(name: &str) -> bool { let name_lc = name.to_ascii_lowercase(); matches!(name_lc.as_str(), "content-length") || is_hop_by_hop_header(&name_lc) @@ -2792,48 +2773,102 @@ mod tests { ); } - #[test] - fn sanitize_request_headers_strips_auth_and_framing() { - let headers = vec![ - ("authorization".to_string(), "Bearer test".to_string()), - ("x-api-key".to_string(), "secret".to_string()), - ("transfer-encoding".to_string(), "chunked".to_string()), - ("content-length".to_string(), "42".to_string()), - ("content-type".to_string(), "application/json".to_string()), - ("accept".to_string(), "text/event-stream".to_string()), - ]; + #[tokio::test] + async fn inference_interception_applies_router_header_allowlist() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::TcpListener; - let kept = sanitize_inference_request_headers(&headers); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let upstream_addr = listener.local_addr().unwrap(); + let upstream_task = tokio::spawn(async move { + use crate::l7::inference::{ParseResult, try_parse_http_request}; - assert!( - kept.iter() - .all(|(k, _)| !k.eq_ignore_ascii_case("authorization")), - "authorization should be stripped" - ); - assert!( - kept.iter() - .all(|(k, _)| !k.eq_ignore_ascii_case("x-api-key")), - "x-api-key should be stripped" - ); - assert!( - kept.iter() - .all(|(k, _)| !k.eq_ignore_ascii_case("transfer-encoding")), - "transfer-encoding should be stripped" - ); - assert!( - kept.iter() - .all(|(k, _)| !k.eq_ignore_ascii_case("content-length")), - "content-length should be stripped" + let (mut upstream, _) = listener.accept().await.unwrap(); + let mut buf = Vec::new(); + let mut chunk = [0u8; 4096]; + + loop { + let n = upstream.read(&mut chunk).await.unwrap(); + assert!(n > 0, "upstream request closed before request completed"); + buf.extend_from_slice(&chunk[..n]); + + match try_parse_http_request(&buf) { + ParseResult::Complete(_, consumed) => { + upstream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") + .await + .unwrap(); + return String::from_utf8_lossy(&buf[..consumed]).to_string(); + } + ParseResult::Incomplete => continue, + ParseResult::Invalid(reason) => { + panic!("forwarded request should parse cleanly: {reason}"); + } + } + } + }); + + let router = openshell_router::Router::new().unwrap(); + let patterns = crate::l7::inference::default_patterns(); + let ctx = InferenceContext::new( + patterns, + router, + vec![openshell_router::config::ResolvedRoute { + name: "inference.local".to_string(), + endpoint: format!("http://{upstream_addr}"), + model: "meta/llama-3.1-8b-instruct".to_string(), + api_key: "test-api-key".to_string(), + protocols: vec!["openai_chat_completions".to_string()], + auth: openshell_router::config::AuthHeader::Bearer, + default_headers: vec![], + passthrough_headers: vec![ + "openai-organization".to_string(), + "x-model-id".to_string(), + ], + timeout: openshell_router::config::DEFAULT_ROUTE_TIMEOUT, + }], + vec![], ); - assert!( - kept.iter() - .any(|(k, _)| k.eq_ignore_ascii_case("content-type")), - "content-type should be preserved" + + let body = r#"{"model":"ignored","messages":[{"role":"user","content":"hi"}]}"#; + let request = format!( + "POST /v1/chat/completions HTTP/1.1\r\n\ + Host: inference.local\r\n\ + Content-Type: application/json\r\n\ + OpenAI-Organization: org_123\r\n\ + Authorization: Bearer client-key\r\n\ + Cookie: session=abc\r\n\ + Content-Length: {}\r\n\r\n{}", + body.len(), + body, ); + + let (client, mut server) = tokio::io::duplex(65536); + let (mut client_read, mut client_write) = tokio::io::split(client); + + let server_task = + tokio::spawn(async move { process_inference_keepalive(&mut server, &ctx, 443).await }); + + client_write.write_all(request.as_bytes()).await.unwrap(); + client_write.shutdown().await.unwrap(); + + let mut response = Vec::new(); + client_read.read_to_end(&mut response).await.unwrap(); + let response_text = String::from_utf8_lossy(&response); + assert!(response_text.starts_with("HTTP/1.1 200")); + + let outcome = server_task.await.unwrap().unwrap(); assert!( - kept.iter().any(|(k, _)| k.eq_ignore_ascii_case("accept")), - "accept should be preserved" + matches!(outcome, InferenceOutcome::Routed), + "expected Routed outcome, got: {outcome:?}" ); + + let forwarded = upstream_task.await.unwrap(); + let forwarded_lc = forwarded.to_ascii_lowercase(); + assert!(forwarded_lc.contains("openai-organization: org_123")); + assert!(forwarded_lc.contains("authorization: bearer test-api-key")); + assert!(!forwarded_lc.contains("authorization: bearer client-key")); + assert!(!forwarded_lc.contains("cookie:")); } // -- router_error_to_http -- diff --git a/crates/openshell-sandbox/tests/system_inference.rs b/crates/openshell-sandbox/tests/system_inference.rs index 5d581fbe23..20c39f3b66 100644 --- a/crates/openshell-sandbox/tests/system_inference.rs +++ b/crates/openshell-sandbox/tests/system_inference.rs @@ -20,6 +20,7 @@ fn make_system_route() -> ResolvedRoute { protocols: vec!["openai_chat_completions".to_string()], auth: AuthHeader::Bearer, default_headers: Vec::new(), + passthrough_headers: Vec::new(), timeout: openshell_router::config::DEFAULT_ROUTE_TIMEOUT, } } @@ -33,6 +34,7 @@ fn make_user_route() -> ResolvedRoute { protocols: vec!["openai_chat_completions".to_string()], auth: AuthHeader::Bearer, default_headers: Vec::new(), + passthrough_headers: Vec::new(), timeout: openshell_router::config::DEFAULT_ROUTE_TIMEOUT, } } @@ -126,6 +128,10 @@ async fn system_inference_with_anthropic_protocol() { protocols: vec!["anthropic_messages".to_string()], auth: AuthHeader::Custom("x-api-key"), default_headers: vec![("anthropic-version".to_string(), "2023-06-01".to_string())], + passthrough_headers: vec![ + "anthropic-version".to_string(), + "anthropic-beta".to_string(), + ], timeout: openshell_router::config::DEFAULT_ROUTE_TIMEOUT, }; diff --git a/crates/openshell-server/src/inference.rs b/crates/openshell-server/src/inference.rs index 0fb29bde55..79f303aebb 100644 --- a/crates/openshell-server/src/inference.rs +++ b/crates/openshell-server/src/inference.rs @@ -276,6 +276,11 @@ fn resolve_provider_route(provider: &Provider) -> Result Date: Thu, 16 Apr 2026 22:40:25 +0200 Subject: [PATCH 003/142] feat(sandbox): load system CA certificates for upstream TLS connections (#862) The proxy's upstream TLS client only trusted Mozilla root CAs (webpki-roots), which prevented TLS termination from working with internal/corporate hosts using private CA certificates. Load system CA certificates from the container's trust store (e.g. /etc/ssl/certs/ca-certificates.crt) in addition to webpki-roots. This allows custom sandbox images to include corporate CAs via update-ca-certificates. Signed-off-by: Matthias Osswald --- architecture/gateway-security.md | 2 +- architecture/sandbox.md | 4 +- crates/openshell-sandbox/src/l7/tls.rs | 157 +++++++++++++++++++++++-- crates/openshell-sandbox/src/lib.rs | 8 +- 4 files changed, 158 insertions(+), 13 deletions(-) diff --git a/architecture/gateway-security.md b/architecture/gateway-security.md index 4989f69b6f..319800c081 100644 --- a/architecture/gateway-security.md +++ b/architecture/gateway-security.md @@ -425,7 +425,7 @@ The sandbox proxy automatically detects and terminates TLS on outbound HTTPS con 1. **Ephemeral sandbox CA**: a per-sandbox CA (`CN=OpenShell Sandbox CA, O=OpenShell`) is generated at sandbox startup. This CA is completely independent of the cluster mTLS CA. 2. **Trust injection**: the sandbox CA is written to the sandbox filesystem and injected via `NODE_EXTRA_CA_CERTS` and `SSL_CERT_FILE` so processes inside the sandbox trust it. 3. **Dynamic leaf certs**: for each target hostname, the proxy generates and caches a leaf certificate signed by the sandbox CA (up to 256 entries). -4. **Upstream verification**: the proxy verifies upstream server certificates against Mozilla root CAs (`webpki-roots`), not against the cluster CA. +4. **Upstream verification**: the proxy verifies upstream server certificates against Mozilla root CAs (`webpki-roots`) and system CA certificates from the container's trust store, not against the cluster CA. Custom sandbox images can add corporate/internal CAs via `update-ca-certificates`. This capability is orthogonal to gateway mTLS -- it operates only on sandbox-to-internet traffic and uses entirely separate key material. See [Policy Language](security-policy.md) for configuration details. diff --git a/architecture/sandbox.md b/architecture/sandbox.md index c6e715f259..c7e789caeb 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -94,7 +94,7 @@ flowchart TD - Generate ephemeral CA via `SandboxCa::generate()` using `rcgen` - Write CA cert PEM and combined bundle (system CAs + sandbox CA) to `/etc/openshell-tls/` - Add the TLS directory to `policy.filesystem.read_only` so Landlock allows the child to read it - - Build upstream `ClientConfig` with Mozilla root CAs via `webpki_roots` + - Build upstream `ClientConfig` with Mozilla root CAs (`webpki_roots`) plus system CA certificates from the container's trust store (e.g. corporate CAs added via `update-ca-certificates`) - Create `Arc` wrapping a `CertCache` and the upstream config 6. **Network namespace** (Linux, proxy mode only): @@ -1057,7 +1057,7 @@ TLS termination is automatic. The proxy peeks the first bytes of every CONNECT t **Connection flow (when TLS is detected):** 1. `tls_terminate_client()`: Accept TLS from the sandboxed client using a `ServerConfig` with the hostname-specific leaf cert. ALPN: `http/1.1`. -2. `tls_connect_upstream()`: Connect TLS to the real upstream using a `ClientConfig` with Mozilla root CAs (`webpki_roots`). ALPN: `http/1.1`. +2. `tls_connect_upstream()`: Connect TLS to the real upstream using a `ClientConfig` with Mozilla root CAs (`webpki_roots`) and system CA certificates. ALPN: `http/1.1`. 3. Proxy now holds plaintext on both sides. If L7 config is present, runs `relay_with_inspection()`. Otherwise, runs `relay_passthrough_with_credentials()` for credential injection without L7 evaluation. System CA bundles are searched at well-known paths: `/etc/ssl/certs/ca-certificates.crt` (Debian/Ubuntu), `/etc/pki/tls/certs/ca-bundle.crt` (RHEL), `/etc/ssl/ca-bundle.pem` (openSUSE), `/etc/ssl/cert.pem` (Alpine/macOS). diff --git a/crates/openshell-sandbox/src/l7/tls.rs b/crates/openshell-sandbox/src/l7/tls.rs index 4ec0de03cc..a11674da12 100644 --- a/crates/openshell-sandbox/src/l7/tls.rs +++ b/crates/openshell-sandbox/src/l7/tls.rs @@ -197,11 +197,28 @@ pub async fn tls_connect_upstream( Ok(tls_stream) } -/// Build a rustls `ClientConfig` with Mozilla root CAs for upstream connections. -pub fn build_upstream_client_config() -> Arc { +/// Build a rustls `ClientConfig` with Mozilla + system root CAs for upstream connections. +/// +/// `system_ca_bundle` is the pre-read PEM contents of the system CA bundle +/// (from [`read_system_ca_bundle`]). Pass the same string to [`write_ca_files`] +/// to avoid reading the bundle from disk twice. +pub fn build_upstream_client_config(system_ca_bundle: &str) -> Arc { let mut root_store = rustls::RootCertStore::empty(); root_store.extend(webpki_roots::TLS_SERVER_ROOTS.iter().cloned()); + // System bundles typically overlap with webpki-roots (Mozilla roots); + // duplicates are harmless and ensure we also pick up any custom/corporate CAs. + let (added, ignored) = load_pem_certs_into_store(&mut root_store, system_ca_bundle); + if added > 0 { + tracing::debug!(added, "Loaded system CA certificates for upstream TLS"); + } + if ignored > 0 { + tracing::warn!( + ignored, + "Some system CA certificates could not be parsed and were ignored" + ); + } + let mut config = ClientConfig::builder() .with_root_certificates(root_store) .with_no_client_auth(); @@ -216,15 +233,23 @@ pub fn build_upstream_client_config() -> Arc { /// 1. Standalone CA cert PEM (for `NODE_EXTRA_CA_CERTS` which is additive) /// 2. Combined bundle: system CAs + sandbox CA (for `SSL_CERT_FILE` which replaces default) /// +/// `system_ca_bundle` is the pre-read PEM contents of the system CA bundle +/// (from [`read_system_ca_bundle`]). Pass the same string to +/// [`build_upstream_client_config`] to avoid reading the bundle from disk twice. +/// /// Returns `(ca_cert_path, combined_bundle_path)`. -pub fn write_ca_files(ca: &SandboxCa, output_dir: &Path) -> Result<(PathBuf, PathBuf)> { +pub fn write_ca_files( + ca: &SandboxCa, + output_dir: &Path, + system_ca_bundle: &str, +) -> Result<(PathBuf, PathBuf)> { std::fs::create_dir_all(output_dir).into_diagnostic()?; let ca_cert_path = output_dir.join("openshell-ca.pem"); std::fs::write(&ca_cert_path, ca.cert_pem()).into_diagnostic()?; - // Read system CA bundle and append our CA - let mut combined = read_system_ca_bundle(); + // Combine system CAs with our sandbox CA + let mut combined = system_ca_bundle.to_string(); if !combined.is_empty() && !combined.ends_with('\n') { combined.push('\n'); } @@ -236,8 +261,36 @@ pub fn write_ca_files(ca: &SandboxCa, output_dir: &Path) -> Result<(PathBuf, Pat Ok((ca_cert_path, combined_path)) } +/// Load PEM-encoded certificates from a string into a root certificate store. +/// +/// Returns `(added, ignored)` counts. Invalid or unparseable certificates +/// are silently ignored, matching the behavior of +/// `RootCertStore::add_parsable_certificates`. +fn load_pem_certs_into_store( + root_store: &mut rustls::RootCertStore, + pem_data: &str, +) -> (usize, usize) { + if pem_data.is_empty() { + return (0, 0); + } + let mut reader = BufReader::new(pem_data.as_bytes()); + // Collect all results so we can count PEM blocks that fail base64 + // decoding — rustls_pemfile::certs silently drops those, so without + // this they wouldn't be reflected in the `ignored` count. + let all_results: Vec<_> = rustls_pemfile::certs(&mut reader).collect(); + let pem_errors = all_results.iter().filter(|r| r.is_err()).count(); + let certs: Vec> = + all_results.into_iter().filter_map(Result::ok).collect(); + let (added, ignored) = root_store.add_parsable_certificates(certs); + (added, ignored + pem_errors) +} + /// Read the system CA bundle from well-known paths. -fn read_system_ca_bundle() -> String { +/// +/// Returns the PEM contents of the first non-empty bundle found, or an empty +/// string if none of the well-known paths exist. Call once and pass the result +/// to both [`write_ca_files`] and [`build_upstream_client_config`]. +pub fn read_system_ca_bundle() -> String { for path in SYSTEM_CA_PATHS { if let Ok(contents) = std::fs::read_to_string(path) && !contents.is_empty() @@ -373,7 +426,97 @@ mod tests { #[test] fn upstream_config_alpn() { let _ = rustls::crypto::ring::default_provider().install_default(); - let config = build_upstream_client_config(); + let config = build_upstream_client_config(""); assert_eq!(config.alpn_protocols, vec![b"http/1.1".to_vec()]); } + + /// Helper: generate a self-signed CA and return its PEM string. + fn generate_ca_pem() -> String { + SandboxCa::generate().unwrap().ca_cert_pem + } + + #[test] + fn load_pem_certs_single_ca() { + let pem = generate_ca_pem(); + let mut store = rustls::RootCertStore::empty(); + let (added, ignored) = load_pem_certs_into_store(&mut store, &pem); + assert_eq!(added, 1); + assert_eq!(ignored, 0); + } + + #[test] + fn load_pem_certs_multiple_cas() { + let bundle = format!( + "{}\n{}\n{}\n", + generate_ca_pem(), + generate_ca_pem(), + generate_ca_pem() + ); + let mut store = rustls::RootCertStore::empty(); + let (added, ignored) = load_pem_certs_into_store(&mut store, &bundle); + assert_eq!(added, 3); + assert_eq!(ignored, 0); + } + + #[test] + fn load_pem_certs_empty_string() { + let mut store = rustls::RootCertStore::empty(); + let (added, ignored) = load_pem_certs_into_store(&mut store, ""); + assert_eq!(added, 0); + assert_eq!(ignored, 0); + } + + #[test] + fn load_pem_certs_garbage_input() { + let mut store = rustls::RootCertStore::empty(); + let (added, ignored) = load_pem_certs_into_store(&mut store, "this is not PEM data at all"); + assert_eq!(added, 0); + assert_eq!(ignored, 0); + } + + #[test] + fn load_pem_certs_malformed_pem_block() { + let malformed = "-----BEGIN CERTIFICATE-----\nNOTBASE64!!!\n-----END CERTIFICATE-----\n"; + let mut store = rustls::RootCertStore::empty(); + let (added, ignored) = load_pem_certs_into_store(&mut store, malformed); + assert_eq!(added, 0); + assert_eq!(ignored, 1); + } + + #[test] + fn load_pem_certs_mixed_valid_and_invalid() { + let malformed = "-----BEGIN CERTIFICATE-----\nNOTBASE64!!!\n-----END CERTIFICATE-----\n"; + let bundle = format!( + "{}\n{}{}\n", + generate_ca_pem(), + malformed, + generate_ca_pem() + ); + let mut store = rustls::RootCertStore::empty(); + let (added, ignored) = load_pem_certs_into_store(&mut store, &bundle); + assert_eq!(added, 2); + assert_eq!(ignored, 1); + } + + #[test] + fn write_ca_files_includes_sandbox_ca() { + let ca = SandboxCa::generate().unwrap(); + let dir = tempfile::tempdir().unwrap(); + let (ca_path, bundle_path) = write_ca_files(&ca, dir.path(), "").unwrap(); + + // Standalone CA cert file should exist and be valid PEM + let ca_pem = std::fs::read_to_string(&ca_path).unwrap(); + assert!(ca_pem.starts_with("-----BEGIN CERTIFICATE-----")); + + // Combined bundle should contain at least the sandbox CA + let bundle_pem = std::fs::read_to_string(&bundle_path).unwrap(); + assert!(bundle_pem.contains(ca.cert_pem())); + + // Bundle should be parseable as PEM certificates + let mut reader = BufReader::new(bundle_pem.as_bytes()); + assert!( + rustls_pemfile::certs(&mut reader).any(|r| r.is_ok()), + "bundle should contain at least one cert", + ); + } } diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index b81dd4a6c2..1fbbe90d42 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -87,7 +87,8 @@ pub(crate) fn ocsf_ctx() -> &'static SandboxContext { use crate::identity::BinaryIdentityCache; use crate::l7::tls::{ - CertCache, ProxyTlsState, SandboxCa, build_upstream_client_config, write_ca_files, + CertCache, ProxyTlsState, SandboxCa, build_upstream_client_config, read_system_ca_bundle, + write_ca_files, }; use crate::opa::OpaEngine; use crate::policy::{NetworkMode, NetworkPolicy, ProxyPolicy, SandboxPolicy}; @@ -315,13 +316,14 @@ pub async fn run_sandbox( match SandboxCa::generate() { Ok(ca) => { let tls_dir = std::path::Path::new("/etc/openshell-tls"); - match write_ca_files(&ca, tls_dir) { + let system_ca_bundle = read_system_ca_bundle(); + match write_ca_files(&ca, tls_dir, &system_ca_bundle) { Ok(paths) => { // /etc/openshell-tls is subsumed by the /etc baseline // path injected by enrich_*_baseline_paths(), so no // explicit Landlock entry is needed here. - let upstream_config = build_upstream_client_config(); + let upstream_config = build_upstream_client_config(&system_ca_bundle); let cert_cache = CertCache::new(ca); let state = Arc::new(ProxyTlsState::new(cert_cache, upstream_config)); ocsf_emit!( From 5718553b9782c6c9ab5ea4c339c72b6f363b4b65 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 16 Apr 2026 16:58:24 -0700 Subject: [PATCH 004/142] feat(release): publish standalone openshell-gateway binaries (#853) --- .github/workflows/docker-build.yml | 4 +- .github/workflows/release-dev.yml | 289 ++++++++++++++++- .github/workflows/release-tag.yml | 292 +++++++++++++++++- architecture/build-containers.md | 14 +- crates/openshell-server/Cargo.toml | 2 +- crates/openshell-server/src/cli.rs | 247 +++++++++++++++ crates/openshell-server/src/lib.rs | 1 + crates/openshell-server/src/main.rs | 216 +------------ .../src/persistence/postgres.rs | 11 +- .../src/persistence/sqlite.rs | 11 +- .../openshell-server/src/persistence/tests.rs | 10 + deploy/docker/Dockerfile.gateway-macos | 105 +++++++ deploy/docker/Dockerfile.images | 29 +- docs/reference/support-matrix.mdx | 14 +- tasks/docker.toml | 26 +- tasks/scripts/docker-build-image.sh | 7 +- 16 files changed, 1003 insertions(+), 275 deletions(-) create mode 100644 crates/openshell-server/src/cli.rs create mode 100644 deploy/docker/Dockerfile.gateway-macos diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 16a8447c99..69a8d1d178 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -4,7 +4,7 @@ on: workflow_call: inputs: component: - description: "Component to build (gateway, cluster)" + description: "Component to build (gateway, supervisor, cluster)" required: true type: string timeout-minutes: @@ -93,4 +93,4 @@ jobs: # Enable dev-settings feature for test settings (dummy_bool, dummy_int) # used by e2e tests. EXTRA_CARGO_FEATURES: openshell-core/dev-settings - run: mise run --no-prepare docker:build:${{ inputs.component }} + run: mise run --no-prepare build:docker:${{ inputs.component }} diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index bdc8b1a719..fc42794cc9 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -54,6 +54,13 @@ jobs: component: gateway cargo-version: ${{ needs.compute-versions.outputs.cargo_version }} + build-supervisor: + needs: [compute-versions] + uses: ./.github/workflows/docker-build.yml + with: + component: supervisor + cargo-version: ${{ needs.compute-versions.outputs.cargo_version }} + build-cluster: needs: [compute-versions] uses: ./.github/workflows/docker-build.yml @@ -70,7 +77,7 @@ jobs: tag-ghcr-dev: name: Tag GHCR Images as Dev - needs: [build-gateway, build-cluster] + needs: [build-gateway, build-supervisor, build-cluster] runs-on: build-amd64 timeout-minutes: 10 steps: @@ -81,7 +88,7 @@ jobs: run: | set -euo pipefail REGISTRY="ghcr.io/nvidia/openshell" - for component in gateway cluster; do + for component in gateway supervisor cluster; do echo "Tagging ${REGISTRY}/${component}:${{ github.sha }} as dev..." docker buildx imagetools create \ --prefer-index=false \ @@ -282,11 +289,6 @@ jobs: # Override z3-sys default (stdc++) so Rust links the matching runtime. echo "CXXSTDLIB=c++" >> "$GITHUB_ENV" - - name: Scope workspace to CLI crates - run: | - set -euo pipefail - sed -i 's|members = \["crates/\*"\]|members = ["crates/openshell-cli", "crates/openshell-core", "crates/openshell-bootstrap", "crates/openshell-policy", "crates/openshell-prover", "crates/openshell-providers", "crates/openshell-tui"]|' Cargo.toml - - name: Patch workspace version if: needs.compute-versions.outputs.cargo_version != '' run: | @@ -378,12 +380,247 @@ jobs: path: artifacts/*.tar.gz retention-days: 5 + # --------------------------------------------------------------------------- + # Build standalone gateway binaries (Linux GNU — native on each arch) + # --------------------------------------------------------------------------- + build-gateway-binary-linux: + name: Build Gateway Binary (Linux ${{ matrix.arch }}) + needs: [compute-versions] + strategy: + matrix: + include: + - arch: amd64 + runner: build-amd64 + target: x86_64-unknown-linux-gnu + - arch: arm64 + runner: build-arm64 + target: aarch64-unknown-linux-gnu + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + container: + image: ghcr.io/nvidia/openshell/ci:latest + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + options: --privileged + env: + MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SCCACHE_MEMCACHED_ENDPOINT: ${{ vars.SCCACHE_MEMCACHED_ENDPOINT }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Mark workspace safe for git + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Fetch tags + run: git fetch --tags --force + + - name: Install tools + run: mise install + + - name: Cache Rust target and registry + uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 + with: + shared-key: gateway-binary-gnu-${{ matrix.arch }} + cache-directories: .cache/sccache + cache-targets: "true" + + - name: Patch workspace version + if: needs.compute-versions.outputs.cargo_version != '' + run: | + set -euo pipefail + sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${{ needs.compute-versions.outputs.cargo_version }}"'"/}' Cargo.toml + + - name: Build ${{ matrix.target }} + run: | + set -euo pipefail + mise x -- cargo build --release --target ${{ matrix.target }} -p openshell-server + + - name: Verify packaged binary + run: | + set -euo pipefail + OUTPUT="$(target/${{ matrix.target }}/release/openshell-gateway --version)" + echo "$OUTPUT" + grep -q '^openshell-gateway ' <<<"$OUTPUT" + + - name: sccache stats + if: always() + run: mise x -- sccache --show-stats + + - name: Package binary + run: | + set -euo pipefail + mkdir -p artifacts + tar -czf artifacts/openshell-gateway-${{ matrix.target }}.tar.gz \ + -C target/${{ matrix.target }}/release openshell-gateway + ls -lh artifacts/ + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: gateway-binary-linux-${{ matrix.arch }} + path: artifacts/*.tar.gz + retention-days: 5 + + # --------------------------------------------------------------------------- + # Build standalone gateway binary (macOS aarch64 via osxcross) + # --------------------------------------------------------------------------- + build-gateway-binary-macos: + name: Build Gateway Binary (macOS) + needs: [compute-versions] + runs-on: build-amd64 + timeout-minutes: 60 + container: + image: ghcr.io/nvidia/openshell/ci:latest + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + options: --privileged + volumes: + - /var/run/docker.sock:/var/run/docker.sock + env: + MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SCCACHE_MEMCACHED_ENDPOINT: ${{ vars.SCCACHE_MEMCACHED_ENDPOINT }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Mark workspace safe for git + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Fetch tags + run: git fetch --tags --force + + - name: Log in to GHCR + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin + + - name: Set up Docker Buildx + uses: ./.github/actions/setup-buildx + + - name: Build macOS binary via Docker + run: | + set -euo pipefail + docker buildx build \ + --file deploy/docker/Dockerfile.gateway-macos \ + --build-arg OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" \ + --build-arg CARGO_TARGET_CACHE_SCOPE="${{ github.sha }}" \ + --target binary \ + --output type=local,dest=out/ \ + . + + - name: Verify packaged binary shape + run: | + set -euo pipefail + test -x out/openshell-gateway + + - name: Package binary + run: | + set -euo pipefail + mkdir -p artifacts + tar -czf artifacts/openshell-gateway-aarch64-apple-darwin.tar.gz \ + -C out openshell-gateway + ls -lh artifacts/ + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: gateway-binary-macos + path: artifacts/*.tar.gz + retention-days: 5 + + # --------------------------------------------------------------------------- + # Build standalone supervisor binaries (Linux GNU — native on each arch) + # --------------------------------------------------------------------------- + build-supervisor-binary-linux: + name: Build Supervisor Binary (Linux ${{ matrix.arch }}) + needs: [compute-versions] + strategy: + matrix: + include: + - arch: amd64 + runner: build-amd64 + target: x86_64-unknown-linux-gnu + - arch: arm64 + runner: build-arm64 + target: aarch64-unknown-linux-gnu + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + container: + image: ghcr.io/nvidia/openshell/ci:latest + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + options: --privileged + env: + MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SCCACHE_MEMCACHED_ENDPOINT: ${{ vars.SCCACHE_MEMCACHED_ENDPOINT }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Mark workspace safe for git + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Fetch tags + run: git fetch --tags --force + + - name: Install tools + run: mise install + + - name: Cache Rust target and registry + uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 + with: + shared-key: supervisor-binary-gnu-${{ matrix.arch }} + cache-directories: .cache/sccache + cache-targets: "true" + + - name: Patch workspace version + if: needs.compute-versions.outputs.cargo_version != '' + run: | + set -euo pipefail + sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${{ needs.compute-versions.outputs.cargo_version }}"'"/}' Cargo.toml + + - name: Build ${{ matrix.target }} + run: | + set -euo pipefail + mise x -- cargo build --release --target ${{ matrix.target }} -p openshell-sandbox --bin openshell-sandbox + + - name: Verify packaged binary + run: | + set -euo pipefail + OUTPUT="$(target/${{ matrix.target }}/release/openshell-sandbox --version)" + echo "$OUTPUT" + grep -q '^openshell-sandbox ' <<<"$OUTPUT" + + - name: sccache stats + if: always() + run: mise x -- sccache --show-stats + + - name: Package binary + run: | + set -euo pipefail + mkdir -p artifacts + tar -czf artifacts/openshell-sandbox-${{ matrix.target }}.tar.gz \ + -C target/${{ matrix.target }}/release openshell-sandbox + ls -lh artifacts/ + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: supervisor-binary-linux-${{ matrix.arch }} + path: artifacts/*.tar.gz + retention-days: 5 + # --------------------------------------------------------------------------- # Create / update the dev GitHub Release with CLI binaries and wheels # --------------------------------------------------------------------------- release-dev: name: Release Dev - needs: [compute-versions, build-cli-linux, build-cli-macos, build-python-wheels-linux, build-python-wheel-macos] + needs: [compute-versions, build-cli-linux, build-cli-macos, build-gateway-binary-linux, build-gateway-binary-macos, build-supervisor-binary-linux, build-python-wheels-linux, build-python-wheel-macos] runs-on: build-amd64 timeout-minutes: 10 outputs: @@ -398,6 +635,20 @@ jobs: path: release/ merge-multiple: true + - name: Download gateway binary artifacts + uses: actions/download-artifact@v4 + with: + pattern: gateway-binary-* + path: release/ + merge-multiple: true + + - name: Download supervisor binary artifacts + uses: actions/download-artifact@v4 + with: + pattern: supervisor-binary-* + path: release/ + merge-multiple: true + - name: Download wheel artifacts uses: actions/download-artifact@v4 with: @@ -417,8 +668,21 @@ jobs: run: | set -euo pipefail cd release - sha256sum *.tar.gz *.whl > openshell-checksums-sha256.txt + sha256sum \ + openshell-x86_64-unknown-linux-musl.tar.gz \ + openshell-aarch64-unknown-linux-musl.tar.gz \ + openshell-aarch64-apple-darwin.tar.gz \ + *.whl > openshell-checksums-sha256.txt cat openshell-checksums-sha256.txt + sha256sum \ + openshell-gateway-x86_64-unknown-linux-gnu.tar.gz \ + openshell-gateway-aarch64-unknown-linux-gnu.tar.gz \ + openshell-gateway-aarch64-apple-darwin.tar.gz > openshell-gateway-checksums-sha256.txt + cat openshell-gateway-checksums-sha256.txt + sha256sum \ + openshell-sandbox-x86_64-unknown-linux-gnu.tar.gz \ + openshell-sandbox-aarch64-unknown-linux-gnu.tar.gz > openshell-sandbox-checksums-sha256.txt + cat openshell-sandbox-checksums-sha256.txt - name: Prune stale wheel assets from dev release uses: actions/github-script@v7 @@ -496,8 +760,15 @@ jobs: release/openshell-x86_64-unknown-linux-musl.tar.gz release/openshell-aarch64-unknown-linux-musl.tar.gz release/openshell-aarch64-apple-darwin.tar.gz + release/openshell-gateway-x86_64-unknown-linux-gnu.tar.gz + release/openshell-gateway-aarch64-unknown-linux-gnu.tar.gz + release/openshell-gateway-aarch64-apple-darwin.tar.gz + release/openshell-sandbox-x86_64-unknown-linux-gnu.tar.gz + release/openshell-sandbox-aarch64-unknown-linux-gnu.tar.gz release/*.whl release/openshell-checksums-sha256.txt + release/openshell-gateway-checksums-sha256.txt + release/openshell-sandbox-checksums-sha256.txt trigger-wheel-publish: name: Trigger Wheel Publish diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index d89d67032b..9b6a940659 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -69,6 +69,13 @@ jobs: component: gateway cargo-version: ${{ needs.compute-versions.outputs.cargo_version }} + build-supervisor: + needs: [compute-versions] + uses: ./.github/workflows/docker-build.yml + with: + component: supervisor + cargo-version: ${{ needs.compute-versions.outputs.cargo_version }} + build-cluster: needs: [compute-versions] uses: ./.github/workflows/docker-build.yml @@ -85,7 +92,7 @@ jobs: tag-ghcr-release: name: Tag GHCR Images for Release - needs: [compute-versions, build-gateway, build-cluster, e2e] + needs: [compute-versions, build-gateway, build-supervisor, build-cluster, e2e] runs-on: build-amd64 timeout-minutes: 10 steps: @@ -97,7 +104,7 @@ jobs: set -euo pipefail REGISTRY="ghcr.io/nvidia/openshell" VERSION="${{ needs.compute-versions.outputs.semver }}" - for component in gateway cluster; do + for component in gateway supervisor cluster; do echo "Tagging ${REGISTRY}/${component}:${{ github.sha }} as ${VERSION} and latest..." docker buildx imagetools create \ --prefer-index=false \ @@ -305,11 +312,6 @@ jobs: # Override z3-sys default (stdc++) so Rust links the matching runtime. echo "CXXSTDLIB=c++" >> "$GITHUB_ENV" - - name: Scope workspace to CLI crates - run: | - set -euo pipefail - sed -i 's|members = \["crates/\*"\]|members = ["crates/openshell-cli", "crates/openshell-core", "crates/openshell-bootstrap", "crates/openshell-policy", "crates/openshell-prover", "crates/openshell-providers", "crates/openshell-tui"]|' Cargo.toml - - name: Patch workspace version if: needs.compute-versions.outputs.cargo_version != '' run: | @@ -402,12 +404,250 @@ jobs: path: artifacts/*.tar.gz retention-days: 5 + # --------------------------------------------------------------------------- + # Build standalone gateway binaries (Linux GNU — native on each arch) + # --------------------------------------------------------------------------- + build-gateway-binary-linux: + name: Build Gateway Binary (Linux ${{ matrix.arch }}) + needs: [compute-versions] + strategy: + matrix: + include: + - arch: amd64 + runner: build-amd64 + target: x86_64-unknown-linux-gnu + - arch: arm64 + runner: build-arm64 + target: aarch64-unknown-linux-gnu + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + container: + image: ghcr.io/nvidia/openshell/ci:latest + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + options: --privileged + env: + MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SCCACHE_MEMCACHED_ENDPOINT: ${{ vars.SCCACHE_MEMCACHED_ENDPOINT }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.tag || github.ref }} + fetch-depth: 0 + + - name: Mark workspace safe for git + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Fetch tags + run: git fetch --tags --force + + - name: Install tools + run: mise install + + - name: Cache Rust target and registry + uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 + with: + shared-key: gateway-binary-gnu-${{ matrix.arch }} + cache-directories: .cache/sccache + cache-targets: "true" + + - name: Patch workspace version + if: needs.compute-versions.outputs.cargo_version != '' + run: | + set -euo pipefail + sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${{ needs.compute-versions.outputs.cargo_version }}"'"/}' Cargo.toml + + - name: Build ${{ matrix.target }} + run: | + set -euo pipefail + mise x -- cargo build --release --target ${{ matrix.target }} -p openshell-server + + - name: Verify packaged binary + run: | + set -euo pipefail + OUTPUT="$(target/${{ matrix.target }}/release/openshell-gateway --version)" + echo "$OUTPUT" + grep -q '^openshell-gateway ' <<<"$OUTPUT" + + - name: sccache stats + if: always() + run: mise x -- sccache --show-stats + + - name: Package binary + run: | + set -euo pipefail + mkdir -p artifacts + tar -czf artifacts/openshell-gateway-${{ matrix.target }}.tar.gz \ + -C target/${{ matrix.target }}/release openshell-gateway + ls -lh artifacts/ + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: gateway-binary-linux-${{ matrix.arch }} + path: artifacts/*.tar.gz + retention-days: 5 + + # --------------------------------------------------------------------------- + # Build standalone supervisor binaries (Linux GNU — native on each arch) + # --------------------------------------------------------------------------- + build-supervisor-binary-linux: + name: Build Supervisor Binary (Linux ${{ matrix.arch }}) + needs: [compute-versions] + strategy: + matrix: + include: + - arch: amd64 + runner: build-amd64 + target: x86_64-unknown-linux-gnu + - arch: arm64 + runner: build-arm64 + target: aarch64-unknown-linux-gnu + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + container: + image: ghcr.io/nvidia/openshell/ci:latest + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + options: --privileged + env: + MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SCCACHE_MEMCACHED_ENDPOINT: ${{ vars.SCCACHE_MEMCACHED_ENDPOINT }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.tag || github.ref }} + fetch-depth: 0 + + - name: Mark workspace safe for git + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Fetch tags + run: git fetch --tags --force + + - name: Install tools + run: mise install + + - name: Cache Rust target and registry + uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 + with: + shared-key: supervisor-binary-gnu-${{ matrix.arch }} + cache-directories: .cache/sccache + cache-targets: "true" + + - name: Patch workspace version + if: needs.compute-versions.outputs.cargo_version != '' + run: | + set -euo pipefail + sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${{ needs.compute-versions.outputs.cargo_version }}"'"/}' Cargo.toml + + - name: Build ${{ matrix.target }} + run: | + set -euo pipefail + mise x -- cargo build --release --target ${{ matrix.target }} -p openshell-sandbox --bin openshell-sandbox + + - name: Verify packaged binary + run: | + set -euo pipefail + OUTPUT="$(target/${{ matrix.target }}/release/openshell-sandbox --version)" + echo "$OUTPUT" + grep -q '^openshell-sandbox ' <<<"$OUTPUT" + + - name: sccache stats + if: always() + run: mise x -- sccache --show-stats + + - name: Package binary + run: | + set -euo pipefail + mkdir -p artifacts + tar -czf artifacts/openshell-sandbox-${{ matrix.target }}.tar.gz \ + -C target/${{ matrix.target }}/release openshell-sandbox + ls -lh artifacts/ + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: supervisor-binary-linux-${{ matrix.arch }} + path: artifacts/*.tar.gz + retention-days: 5 + + # --------------------------------------------------------------------------- + # Build standalone gateway binary (macOS aarch64 via osxcross) + # --------------------------------------------------------------------------- + build-gateway-binary-macos: + name: Build Gateway Binary (macOS) + needs: [compute-versions] + runs-on: build-amd64 + timeout-minutes: 60 + container: + image: ghcr.io/nvidia/openshell/ci:latest + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + options: --privileged + volumes: + - /var/run/docker.sock:/var/run/docker.sock + env: + MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SCCACHE_MEMCACHED_ENDPOINT: ${{ vars.SCCACHE_MEMCACHED_ENDPOINT }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.tag || github.ref }} + fetch-depth: 0 + + - name: Mark workspace safe for git + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Fetch tags + run: git fetch --tags --force + + - name: Log in to GHCR + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin + + - name: Set up Docker Buildx + uses: ./.github/actions/setup-buildx + + - name: Build macOS binary via Docker + run: | + set -euo pipefail + docker buildx build \ + --file deploy/docker/Dockerfile.gateway-macos \ + --build-arg OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" \ + --build-arg CARGO_TARGET_CACHE_SCOPE="${{ github.sha }}" \ + --target binary \ + --output type=local,dest=out/ \ + . + + - name: Verify packaged binary shape + run: | + set -euo pipefail + test -x out/openshell-gateway + + - name: Package binary + run: | + set -euo pipefail + mkdir -p artifacts + tar -czf artifacts/openshell-gateway-aarch64-apple-darwin.tar.gz \ + -C out openshell-gateway + ls -lh artifacts/ + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: gateway-binary-macos + path: artifacts/*.tar.gz + retention-days: 5 + # --------------------------------------------------------------------------- # Create a tagged GitHub Release with CLI binaries and wheels # --------------------------------------------------------------------------- release: name: Release - needs: [compute-versions, build-cli-linux, build-cli-macos, build-python-wheels-linux, build-python-wheel-macos, tag-ghcr-release] + needs: [compute-versions, build-cli-linux, build-cli-macos, build-gateway-binary-linux, build-gateway-binary-macos, build-supervisor-binary-linux, build-python-wheels-linux, build-python-wheel-macos, tag-ghcr-release] runs-on: build-amd64 timeout-minutes: 10 outputs: @@ -424,6 +664,20 @@ jobs: path: release/ merge-multiple: true + - name: Download gateway binary artifacts + uses: actions/download-artifact@v4 + with: + pattern: gateway-binary-* + path: release/ + merge-multiple: true + + - name: Download supervisor binary artifacts + uses: actions/download-artifact@v4 + with: + pattern: supervisor-binary-* + path: release/ + merge-multiple: true + - name: Download wheel artifacts uses: actions/download-artifact@v4 with: @@ -443,8 +697,21 @@ jobs: run: | set -euo pipefail cd release - sha256sum *.tar.gz *.whl > openshell-checksums-sha256.txt + sha256sum \ + openshell-x86_64-unknown-linux-musl.tar.gz \ + openshell-aarch64-unknown-linux-musl.tar.gz \ + openshell-aarch64-apple-darwin.tar.gz \ + *.whl > openshell-checksums-sha256.txt cat openshell-checksums-sha256.txt + sha256sum \ + openshell-gateway-x86_64-unknown-linux-gnu.tar.gz \ + openshell-gateway-aarch64-unknown-linux-gnu.tar.gz \ + openshell-gateway-aarch64-apple-darwin.tar.gz > openshell-gateway-checksums-sha256.txt + cat openshell-gateway-checksums-sha256.txt + sha256sum \ + openshell-sandbox-x86_64-unknown-linux-gnu.tar.gz \ + openshell-sandbox-aarch64-unknown-linux-gnu.tar.gz > openshell-sandbox-checksums-sha256.txt + cat openshell-sandbox-checksums-sha256.txt - name: Create GitHub Release uses: softprops/action-gh-release@v2 @@ -466,8 +733,15 @@ jobs: release/openshell-x86_64-unknown-linux-musl.tar.gz release/openshell-aarch64-unknown-linux-musl.tar.gz release/openshell-aarch64-apple-darwin.tar.gz + release/openshell-gateway-x86_64-unknown-linux-gnu.tar.gz + release/openshell-gateway-aarch64-unknown-linux-gnu.tar.gz + release/openshell-gateway-aarch64-apple-darwin.tar.gz + release/openshell-sandbox-x86_64-unknown-linux-gnu.tar.gz + release/openshell-sandbox-aarch64-unknown-linux-gnu.tar.gz release/*.whl release/openshell-checksums-sha256.txt + release/openshell-gateway-checksums-sha256.txt + release/openshell-sandbox-checksums-sha256.txt publish-fern-docs: name: Publish Fern Docs diff --git a/architecture/build-containers.md b/architecture/build-containers.md index 493e7207a8..196663e7a3 100644 --- a/architecture/build-containers.md +++ b/architecture/build-containers.md @@ -9,7 +9,7 @@ The gateway runs the control plane API server. It is deployed as a StatefulSet i - **Docker target**: `gateway` in `deploy/docker/Dockerfile.images` - **Registry**: `ghcr.io/nvidia/openshell/gateway:latest` - **Pulled when**: Cluster startup (the Helm chart triggers the pull) -- **Entrypoint**: `openshell-server --port 8080` (gRPC + HTTP, mTLS) +- **Entrypoint**: `openshell-gateway --port 8080` (gRPC + HTTP, mTLS) ## Cluster (`openshell/cluster`) @@ -21,6 +21,18 @@ The cluster image is a single-container Kubernetes distribution that bundles the The supervisor binary (`openshell-sandbox`) is built by the shared `supervisor-builder` stage in `deploy/docker/Dockerfile.images` and placed at `/opt/openshell/bin/openshell-sandbox`. It is exposed to sandbox pods at runtime via a read-only `hostPath` volume mount — it is not baked into sandbox images. +## Standalone Gateway Binary + +OpenShell also publishes a standalone `openshell-gateway` binary as a GitHub release asset. + +- **Source crate**: `crates/openshell-server` +- **Artifact name**: `openshell-gateway-.tar.gz` +- **Targets**: `x86_64-unknown-linux-gnu`, `aarch64-unknown-linux-gnu`, `aarch64-apple-darwin` +- **Release workflows**: `.github/workflows/release-dev.yml`, `.github/workflows/release-tag.yml` +- **Installer**: None yet. The binary is a manual-download asset. + +Both the standalone artifact and the deployed container image use the `openshell-gateway` binary. + ## Python Wheels OpenShell also publishes Python wheels for `linux/amd64`, `linux/arm64`, and macOS ARM64. diff --git a/crates/openshell-server/Cargo.toml b/crates/openshell-server/Cargo.toml index b4e8b9e2f5..33e3542475 100644 --- a/crates/openshell-server/Cargo.toml +++ b/crates/openshell-server/Cargo.toml @@ -11,7 +11,7 @@ license.workspace = true repository.workspace = true [[bin]] -name = "openshell-server" +name = "openshell-gateway" path = "src/main.rs" [dependencies] diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs new file mode 100644 index 0000000000..9509fe84b5 --- /dev/null +++ b/crates/openshell-server/src/cli.rs @@ -0,0 +1,247 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Shared CLI entrypoint for the gateway binaries. + +use clap::{Command, CommandFactory, FromArgMatches, Parser}; +use miette::{IntoDiagnostic, Result}; +use openshell_core::ComputeDriverKind; +use std::net::SocketAddr; +use std::path::PathBuf; +use tracing::info; +use tracing_subscriber::EnvFilter; + +use crate::{run_server, tracing_bus::TracingLogBus}; + +/// `OpenShell` gateway process - gRPC and HTTP server with protocol multiplexing. +#[derive(Parser, Debug)] +#[command(version = openshell_core::VERSION)] +#[command(about = "OpenShell gRPC/HTTP server", long_about = None)] +struct Args { + /// Port to bind the server to (all interfaces). + #[arg(long, default_value_t = 8080, env = "OPENSHELL_SERVER_PORT")] + port: u16, + + /// Log level (trace, debug, info, warn, error). + #[arg(long, default_value = "info", env = "OPENSHELL_LOG_LEVEL")] + log_level: String, + + /// Path to TLS certificate file (required unless --disable-tls). + #[arg(long, env = "OPENSHELL_TLS_CERT")] + tls_cert: Option, + + /// Path to TLS private key file (required unless --disable-tls). + #[arg(long, env = "OPENSHELL_TLS_KEY")] + tls_key: Option, + + /// Path to CA certificate for client certificate verification (mTLS). + #[arg(long, env = "OPENSHELL_TLS_CLIENT_CA")] + tls_client_ca: Option, + + /// Database URL for persistence. + #[arg(long, env = "OPENSHELL_DB_URL", required = true)] + db_url: String, + + /// Compute drivers configured for this gateway. + /// + /// Accepts a comma-delimited list such as `kubernetes` or + /// `kubernetes,podman`. The configuration format is future-proofed for + /// multiple drivers, but the gateway currently requires exactly one. + #[arg( + long, + alias = "driver", + env = "OPENSHELL_DRIVERS", + value_delimiter = ',', + default_value = "kubernetes", + value_parser = parse_compute_driver + )] + drivers: Vec, + + /// Kubernetes namespace for sandboxes. + #[arg(long, env = "OPENSHELL_SANDBOX_NAMESPACE", default_value = "default")] + sandbox_namespace: String, + + /// Default container image for sandboxes. + #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE")] + sandbox_image: Option, + + /// Kubernetes imagePullPolicy for sandbox pods (Always, IfNotPresent, Never). + #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY")] + sandbox_image_pull_policy: Option, + + /// gRPC endpoint for sandboxes to callback to `OpenShell`. + /// This should be reachable from within the Kubernetes cluster. + #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] + grpc_endpoint: Option, + + /// Public host for the SSH gateway. + #[arg(long, env = "OPENSHELL_SSH_GATEWAY_HOST", default_value = "127.0.0.1")] + ssh_gateway_host: String, + + /// Public port for the SSH gateway. + #[arg(long, env = "OPENSHELL_SSH_GATEWAY_PORT", default_value_t = 8080)] + ssh_gateway_port: u16, + + /// HTTP path for SSH CONNECT/upgrade. + #[arg( + long, + env = "OPENSHELL_SSH_CONNECT_PATH", + default_value = "/connect/ssh" + )] + ssh_connect_path: String, + + /// SSH port inside sandbox pods. + #[arg(long, env = "OPENSHELL_SANDBOX_SSH_PORT", default_value_t = 2222)] + sandbox_ssh_port: u16, + + /// Shared secret for gateway-to-sandbox SSH handshake. + #[arg(long, env = "OPENSHELL_SSH_HANDSHAKE_SECRET")] + ssh_handshake_secret: Option, + + /// Allowed clock skew in seconds for SSH handshake. + #[arg(long, env = "OPENSHELL_SSH_HANDSHAKE_SKEW_SECS", default_value_t = 300)] + ssh_handshake_skew_secs: u64, + + /// Kubernetes secret name containing client TLS materials for sandbox pods. + #[arg(long, env = "OPENSHELL_CLIENT_TLS_SECRET_NAME")] + client_tls_secret_name: Option, + + /// Host gateway IP for sandbox pod hostAliases. + /// When set, sandbox pods get hostAliases entries mapping + /// host.docker.internal and host.openshell.internal to this IP. + #[arg(long, env = "OPENSHELL_HOST_GATEWAY_IP")] + host_gateway_ip: Option, + + /// Disable TLS entirely — listen on plaintext HTTP. + /// Use this when the gateway sits behind a reverse proxy or tunnel + /// (e.g. Cloudflare Tunnel) that terminates TLS at the edge. + #[arg(long, env = "OPENSHELL_DISABLE_TLS")] + disable_tls: bool, + + /// Disable gateway authentication (mTLS client certificate requirement). + /// When set, the TLS handshake accepts connections without a client + /// certificate. Ignored when --disable-tls is set. + #[arg(long, env = "OPENSHELL_DISABLE_GATEWAY_AUTH")] + disable_gateway_auth: bool, +} + +pub fn command() -> Command { + Args::command() + .name("openshell-gateway") + .bin_name("openshell-gateway") +} + +pub async fn run_cli() -> Result<()> { + rustls::crypto::ring::default_provider() + .install_default() + .map_err(|e| miette::miette!("failed to install rustls crypto provider: {e:?}"))?; + + let args = Args::from_arg_matches(&command().get_matches()).expect("clap validated args"); + + run_from_args(args).await +} + +async fn run_from_args(args: Args) -> Result<()> { + let tracing_log_bus = TracingLogBus::new(); + tracing_log_bus.install_subscriber( + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)), + ); + + let bind = SocketAddr::from(([0, 0, 0, 0], args.port)); + + let tls = if args.disable_tls { + None + } else { + let cert_path = args.tls_cert.ok_or_else(|| { + miette::miette!( + "--tls-cert is required when TLS is enabled (use --disable-tls to skip)" + ) + })?; + let key_path = args.tls_key.ok_or_else(|| { + miette::miette!("--tls-key is required when TLS is enabled (use --disable-tls to skip)") + })?; + let client_ca_path = args.tls_client_ca.ok_or_else(|| { + miette::miette!( + "--tls-client-ca is required when TLS is enabled (use --disable-tls to skip)" + ) + })?; + Some(openshell_core::TlsConfig { + cert_path, + key_path, + client_ca_path, + allow_unauthenticated: args.disable_gateway_auth, + }) + }; + + let mut config = openshell_core::Config::new(tls) + .with_bind_address(bind) + .with_log_level(&args.log_level); + + config = config + .with_database_url(args.db_url) + .with_compute_drivers(args.drivers) + .with_sandbox_namespace(args.sandbox_namespace) + .with_ssh_gateway_host(args.ssh_gateway_host) + .with_ssh_gateway_port(args.ssh_gateway_port) + .with_ssh_connect_path(args.ssh_connect_path) + .with_sandbox_ssh_port(args.sandbox_ssh_port) + .with_ssh_handshake_skew_secs(args.ssh_handshake_skew_secs); + + if let Some(image) = args.sandbox_image { + config = config.with_sandbox_image(image); + } + + if let Some(policy) = args.sandbox_image_pull_policy { + config = config.with_sandbox_image_pull_policy(policy); + } + + if let Some(endpoint) = args.grpc_endpoint { + config = config.with_grpc_endpoint(endpoint); + } + + if let Some(secret) = args.ssh_handshake_secret { + config = config.with_ssh_handshake_secret(secret); + } + + if let Some(name) = args.client_tls_secret_name { + config = config.with_client_tls_secret_name(name); + } + + if let Some(ip) = args.host_gateway_ip { + config = config.with_host_gateway_ip(ip); + } + + if args.disable_tls { + info!("TLS disabled — listening on plaintext HTTP"); + } else if args.disable_gateway_auth { + info!("Gateway auth disabled — accepting connections without client certificates"); + } + + info!(bind = %config.bind_address, "Starting OpenShell server"); + + run_server(config, tracing_log_bus).await.into_diagnostic() +} + +fn parse_compute_driver(value: &str) -> std::result::Result { + value.parse() +} + +#[cfg(test)] +mod tests { + use super::command; + + #[test] + fn command_uses_gateway_binary_name() { + let mut help = Vec::new(); + command().write_long_help(&mut help).unwrap(); + let help = String::from_utf8(help).unwrap(); + assert!(help.contains("openshell-gateway")); + } + + #[test] + fn command_exposes_version() { + let cmd = command(); + let version = cmd.get_version().unwrap(); + assert_eq!(version.to_string(), openshell_core::VERSION); + } +} diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index a8d820b4de..7549a17740 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -10,6 +10,7 @@ //! - mTLS support mod auth; +pub mod cli; mod compute; mod grpc; mod http; diff --git a/crates/openshell-server/src/main.rs b/crates/openshell-server/src/main.rs index ed6c73825a..0f33c685f4 100644 --- a/crates/openshell-server/src/main.rs +++ b/crates/openshell-server/src/main.rs @@ -1,221 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! `OpenShell` Server - gRPC/HTTP server with protocol multiplexing. +//! `OpenShell` Gateway binary entrypoint. -use clap::Parser; -use miette::{IntoDiagnostic, Result}; -use openshell_core::ComputeDriverKind; -use std::net::SocketAddr; -use std::path::PathBuf; -use tracing::info; -use tracing_subscriber::EnvFilter; - -use openshell_server::{run_server, tracing_bus::TracingLogBus}; - -/// `OpenShell` Server - gRPC and HTTP server with protocol multiplexing. -#[derive(Parser, Debug)] -#[command(name = "openshell-server")] -#[command(version = openshell_core::VERSION)] -#[command(about = "OpenShell gRPC/HTTP server", long_about = None)] -struct Args { - /// Port to bind the server to (all interfaces). - #[arg(long, default_value_t = 8080, env = "OPENSHELL_SERVER_PORT")] - port: u16, - - /// Log level (trace, debug, info, warn, error). - #[arg(long, default_value = "info", env = "OPENSHELL_LOG_LEVEL")] - log_level: String, - - /// Path to TLS certificate file (required unless --disable-tls). - #[arg(long, env = "OPENSHELL_TLS_CERT")] - tls_cert: Option, - - /// Path to TLS private key file (required unless --disable-tls). - #[arg(long, env = "OPENSHELL_TLS_KEY")] - tls_key: Option, - - /// Path to CA certificate for client certificate verification (mTLS). - #[arg(long, env = "OPENSHELL_TLS_CLIENT_CA")] - tls_client_ca: Option, - - /// Database URL for persistence. - #[arg(long, env = "OPENSHELL_DB_URL", required = true)] - db_url: String, - - /// Compute drivers configured for this gateway. - /// - /// Accepts a comma-delimited list such as `kubernetes` or - /// `kubernetes,podman`. The configuration format is future-proofed for - /// multiple drivers, but the gateway currently requires exactly one. - #[arg( - long, - alias = "driver", - env = "OPENSHELL_DRIVERS", - value_delimiter = ',', - default_value = "kubernetes", - value_parser = parse_compute_driver - )] - drivers: Vec, - - /// Kubernetes namespace for sandboxes. - #[arg(long, env = "OPENSHELL_SANDBOX_NAMESPACE", default_value = "default")] - sandbox_namespace: String, - - /// Default container image for sandboxes. - #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE")] - sandbox_image: Option, - - /// Kubernetes imagePullPolicy for sandbox pods (Always, IfNotPresent, Never). - #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY")] - sandbox_image_pull_policy: Option, - - /// gRPC endpoint for sandboxes to callback to `OpenShell`. - /// This should be reachable from within the Kubernetes cluster. - #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] - grpc_endpoint: Option, - - /// Public host for the SSH gateway. - #[arg(long, env = "OPENSHELL_SSH_GATEWAY_HOST", default_value = "127.0.0.1")] - ssh_gateway_host: String, - - /// Public port for the SSH gateway. - #[arg(long, env = "OPENSHELL_SSH_GATEWAY_PORT", default_value_t = 8080)] - ssh_gateway_port: u16, - - /// HTTP path for SSH CONNECT/upgrade. - #[arg( - long, - env = "OPENSHELL_SSH_CONNECT_PATH", - default_value = "/connect/ssh" - )] - ssh_connect_path: String, - - /// SSH port inside sandbox pods. - #[arg(long, env = "OPENSHELL_SANDBOX_SSH_PORT", default_value_t = 2222)] - sandbox_ssh_port: u16, - - /// Shared secret for gateway-to-sandbox SSH handshake. - #[arg(long, env = "OPENSHELL_SSH_HANDSHAKE_SECRET")] - ssh_handshake_secret: Option, - - /// Allowed clock skew in seconds for SSH handshake. - #[arg(long, env = "OPENSHELL_SSH_HANDSHAKE_SKEW_SECS", default_value_t = 300)] - ssh_handshake_skew_secs: u64, - - /// Kubernetes secret name containing client TLS materials for sandbox pods. - #[arg(long, env = "OPENSHELL_CLIENT_TLS_SECRET_NAME")] - client_tls_secret_name: Option, - - /// Host gateway IP for sandbox pod hostAliases. - /// When set, sandbox pods get hostAliases entries mapping - /// host.docker.internal and host.openshell.internal to this IP. - #[arg(long, env = "OPENSHELL_HOST_GATEWAY_IP")] - host_gateway_ip: Option, - - /// Disable TLS entirely — listen on plaintext HTTP. - /// Use this when the gateway sits behind a reverse proxy or tunnel - /// (e.g. Cloudflare Tunnel) that terminates TLS at the edge. - #[arg(long, env = "OPENSHELL_DISABLE_TLS")] - disable_tls: bool, - - /// Disable gateway authentication (mTLS client certificate requirement). - /// When set, the TLS handshake accepts connections without a client - /// certificate. Ignored when --disable-tls is set. - #[arg(long, env = "OPENSHELL_DISABLE_GATEWAY_AUTH")] - disable_gateway_auth: bool, -} +use miette::Result; #[tokio::main] async fn main() -> Result<()> { - rustls::crypto::ring::default_provider() - .install_default() - .map_err(|e| miette::miette!("failed to install rustls crypto provider: {e:?}"))?; - - let args = Args::parse(); - - // Initialize tracing - let tracing_log_bus = TracingLogBus::new(); - tracing_log_bus.install_subscriber( - EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)), - ); - - // Build configuration - let bind = SocketAddr::from(([0, 0, 0, 0], args.port)); - - let tls = if args.disable_tls { - None - } else { - let cert_path = args.tls_cert.ok_or_else(|| { - miette::miette!( - "--tls-cert is required when TLS is enabled (use --disable-tls to skip)" - ) - })?; - let key_path = args.tls_key.ok_or_else(|| { - miette::miette!("--tls-key is required when TLS is enabled (use --disable-tls to skip)") - })?; - let client_ca_path = args.tls_client_ca.ok_or_else(|| { - miette::miette!( - "--tls-client-ca is required when TLS is enabled (use --disable-tls to skip)" - ) - })?; - Some(openshell_core::TlsConfig { - cert_path, - key_path, - client_ca_path, - allow_unauthenticated: args.disable_gateway_auth, - }) - }; - - let mut config = openshell_core::Config::new(tls) - .with_bind_address(bind) - .with_log_level(&args.log_level); - - config = config - .with_database_url(args.db_url) - .with_compute_drivers(args.drivers) - .with_sandbox_namespace(args.sandbox_namespace) - .with_ssh_gateway_host(args.ssh_gateway_host) - .with_ssh_gateway_port(args.ssh_gateway_port) - .with_ssh_connect_path(args.ssh_connect_path) - .with_sandbox_ssh_port(args.sandbox_ssh_port) - .with_ssh_handshake_skew_secs(args.ssh_handshake_skew_secs); - - if let Some(image) = args.sandbox_image { - config = config.with_sandbox_image(image); - } - - if let Some(policy) = args.sandbox_image_pull_policy { - config = config.with_sandbox_image_pull_policy(policy); - } - - if let Some(endpoint) = args.grpc_endpoint { - config = config.with_grpc_endpoint(endpoint); - } - - if let Some(secret) = args.ssh_handshake_secret { - config = config.with_ssh_handshake_secret(secret); - } - - if let Some(name) = args.client_tls_secret_name { - config = config.with_client_tls_secret_name(name); - } - - if let Some(ip) = args.host_gateway_ip { - config = config.with_host_gateway_ip(ip); - } - - if args.disable_tls { - info!("TLS disabled — listening on plaintext HTTP"); - } else if args.disable_gateway_auth { - info!("Gateway auth disabled — accepting connections without client certificates"); - } - - info!(bind = %config.bind_address, "Starting OpenShell server"); - - run_server(config, tracing_log_bus).await.into_diagnostic() -} - -fn parse_compute_driver(value: &str) -> std::result::Result { - value.parse() + openshell_server::cli::run_cli().await } diff --git a/crates/openshell-server/src/persistence/postgres.rs b/crates/openshell-server/src/persistence/postgres.rs index 4b62516c4d..509b028d71 100644 --- a/crates/openshell-server/src/persistence/postgres.rs +++ b/crates/openshell-server/src/persistence/postgres.rs @@ -7,7 +7,8 @@ use super::{ use openshell_core::Result; use sqlx::postgres::PgPoolOptions; use sqlx::{PgPool, Row}; -use std::path::PathBuf; + +static POSTGRES_MIGRATOR: sqlx::migrate::Migrator = sqlx::migrate!("./migrations/postgres"); #[derive(Debug, Clone)] pub struct PostgresStore { @@ -26,13 +27,7 @@ impl PostgresStore { } pub async fn migrate(&self) -> Result<()> { - let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("migrations") - .join("postgres"); - let migrator = sqlx::migrate::Migrator::new(path) - .await - .map_err(|e| map_migrate_error(&e))?; - migrator + POSTGRES_MIGRATOR .run(&self.pool) .await .map_err(|e| map_migrate_error(&e)) diff --git a/crates/openshell-server/src/persistence/sqlite.rs b/crates/openshell-server/src/persistence/sqlite.rs index 167f3521db..3ee7799d79 100644 --- a/crates/openshell-server/src/persistence/sqlite.rs +++ b/crates/openshell-server/src/persistence/sqlite.rs @@ -7,9 +7,10 @@ use super::{ use openshell_core::Result; use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions}; use sqlx::{Row, SqlitePool}; -use std::path::PathBuf; use std::str::FromStr; +static SQLITE_MIGRATOR: sqlx::migrate::Migrator = sqlx::migrate!("./migrations/sqlite"); + #[derive(Debug, Clone)] pub struct SqliteStore { pool: SqlitePool, @@ -38,13 +39,7 @@ impl SqliteStore { } pub async fn migrate(&self) -> Result<()> { - let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("migrations") - .join("sqlite"); - let migrator = sqlx::migrate::Migrator::new(path) - .await - .map_err(|e| map_migrate_error(&e))?; - migrator + SQLITE_MIGRATOR .run(&self.pool) .await .map_err(|e| map_migrate_error(&e)) diff --git a/crates/openshell-server/src/persistence/tests.rs b/crates/openshell-server/src/persistence/tests.rs index a5223a9f45..b3bfe2fa42 100644 --- a/crates/openshell-server/src/persistence/tests.rs +++ b/crates/openshell-server/src/persistence/tests.rs @@ -22,6 +22,16 @@ async fn sqlite_put_get_round_trip() { assert_eq!(record.payload, b"payload"); } +#[tokio::test] +async fn sqlite_connect_runs_embedded_migrations() { + let store = Store::connect("sqlite::memory:?cache=shared") + .await + .unwrap(); + + let records = store.list("sandbox", 10, 0).await.unwrap(); + assert!(records.is_empty()); +} + #[tokio::test] async fn sqlite_updates_timestamp() { let store = Store::connect("sqlite::memory:?cache=shared") diff --git a/deploy/docker/Dockerfile.gateway-macos b/deploy/docker/Dockerfile.gateway-macos new file mode 100644 index 0000000000..b0ac282fc5 --- /dev/null +++ b/deploy/docker/Dockerfile.gateway-macos @@ -0,0 +1,105 @@ +# syntax=docker/dockerfile:1.6 + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Cross-compile the standalone openshell-gateway binary for macOS aarch64 +# (Apple Silicon) using the osxcross toolchain. + +ARG OSXCROSS_IMAGE=crazymax/osxcross:latest + +FROM ${OSXCROSS_IMAGE} AS osxcross + +FROM python:3.12-slim AS builder + +ARG CARGO_TARGET_CACHE_SCOPE=default + +ENV PATH="/root/.cargo/bin:/usr/local/bin:/osxcross/bin:${PATH}" +ENV LD_LIBRARY_PATH="/osxcross/lib" + +COPY --from=osxcross /osxcross /osxcross + +RUN SDKROOT="$(echo /osxcross/SDK/MacOSX*.sdk)" && ln -sfn "${SDKROOT}" /osxcross/SDK/MacOSX.sdk + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + clang \ + cmake \ + curl \ + libclang-dev \ + pkg-config \ + && rm -rf /var/lib/apt/lists/* + +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.88.0 + +RUN ln -sf /osxcross/bin/arm64-apple-darwin25.1-ld /usr/local/bin/arm64-apple-macosx-ld + +RUN rustup target add aarch64-apple-darwin + +WORKDIR /build + +ENV CC_aarch64_apple_darwin=oa64-clang +ENV CXX_aarch64_apple_darwin=oa64-clang++ +ENV AR_aarch64_apple_darwin=aarch64-apple-darwin25.1-ar +ENV CARGO_TARGET_AARCH64_APPLE_DARWIN_LINKER=oa64-clang +ENV CARGO_TARGET_AARCH64_APPLE_DARWIN_AR=aarch64-apple-darwin25.1-ar +ENV SDKROOT=/osxcross/SDK/MacOSX.sdk +ENV MACOSX_DEPLOYMENT_TARGET=13.3 +ENV CFLAGS_aarch64_apple_darwin=--target=arm64-apple-macosx\ -mmacosx-version-min=13.3 +ENV CXXFLAGS_aarch64_apple_darwin=--target=arm64-apple-macosx\ -mmacosx-version-min=13.3 +ENV BINDGEN_EXTRA_CLANG_ARGS_aarch64_apple_darwin=--target=arm64-apple-macosx\ -isysroot\ ${SDKROOT} + +COPY Cargo.toml Cargo.lock ./ +COPY crates/openshell-core/Cargo.toml crates/openshell-core/Cargo.toml +COPY crates/openshell-driver-kubernetes/Cargo.toml crates/openshell-driver-kubernetes/Cargo.toml +COPY crates/openshell-policy/Cargo.toml crates/openshell-policy/Cargo.toml +COPY crates/openshell-router/Cargo.toml crates/openshell-router/Cargo.toml +COPY crates/openshell-server/Cargo.toml crates/openshell-server/Cargo.toml +COPY crates/openshell-core/build.rs crates/openshell-core/build.rs +COPY proto/ proto/ + +RUN sed -i 's|members = \["crates/\*"\]|members = ["crates/openshell-server", "crates/openshell-core", "crates/openshell-driver-kubernetes", "crates/openshell-policy", "crates/openshell-router"]|' Cargo.toml + +RUN mkdir -p crates/openshell-core/src \ + crates/openshell-driver-kubernetes/src \ + crates/openshell-policy/src \ + crates/openshell-router/src \ + crates/openshell-server/src && \ + touch crates/openshell-core/src/lib.rs && \ + touch crates/openshell-driver-kubernetes/src/lib.rs && \ + printf 'fn main() {}\n' > crates/openshell-driver-kubernetes/src/main.rs && \ + touch crates/openshell-policy/src/lib.rs && \ + touch crates/openshell-router/src/lib.rs && \ + touch crates/openshell-server/src/lib.rs && \ + printf 'fn main() {}\n' > crates/openshell-server/src/main.rs + +RUN --mount=type=cache,id=cargo-registry-gateway-macos,sharing=locked,target=/root/.cargo/registry \ + --mount=type=cache,id=cargo-git-gateway-macos,sharing=locked,target=/root/.cargo/git \ + --mount=type=cache,id=cargo-target-gateway-macos-${CARGO_TARGET_CACHE_SCOPE},sharing=locked,target=/build/target \ + cargo build --release --target aarch64-apple-darwin -p openshell-server 2>/dev/null || true + +COPY crates/ crates/ + +RUN touch crates/openshell-core/src/lib.rs \ + crates/openshell-driver-kubernetes/src/lib.rs \ + crates/openshell-driver-kubernetes/src/main.rs \ + crates/openshell-policy/src/lib.rs \ + crates/openshell-router/src/lib.rs \ + crates/openshell-server/src/lib.rs \ + crates/openshell-server/src/main.rs \ + crates/openshell-core/build.rs \ + proto/*.proto + +ARG OPENSHELL_CARGO_VERSION +RUN --mount=type=cache,id=cargo-registry-gateway-macos,sharing=locked,target=/root/.cargo/registry \ + --mount=type=cache,id=cargo-git-gateway-macos,sharing=locked,target=/root/.cargo/git \ + --mount=type=cache,id=cargo-target-gateway-macos-${CARGO_TARGET_CACHE_SCOPE},sharing=locked,target=/build/target \ + if [ -n "${OPENSHELL_CARGO_VERSION:-}" ]; then \ + sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${OPENSHELL_CARGO_VERSION}"'"/}' Cargo.toml; \ + fi && \ + cargo build --release --target aarch64-apple-darwin -p openshell-server && \ + cp target/aarch64-apple-darwin/release/openshell-gateway /openshell-gateway + +FROM scratch AS binary +COPY --from=builder /openshell-gateway /openshell-gateway diff --git a/deploy/docker/Dockerfile.images b/deploy/docker/Dockerfile.images index b7e854677d..d29fa3d7f2 100644 --- a/deploy/docker/Dockerfile.images +++ b/deploy/docker/Dockerfile.images @@ -7,8 +7,9 @@ # # Targets: # gateway Final gateway image +# supervisor Final supervisor image # cluster Final cluster image -# gateway-builder Release openshell-server binary +# gateway-builder Release openshell-gateway binary # supervisor-builder Release openshell-sandbox binary # supervisor-output Minimal stage exporting only the supervisor binary @@ -138,7 +139,7 @@ RUN --mount=type=cache,id=cargo-registry-${TARGETARCH},sharing=locked,target=/us . cross-build.sh && \ cargo_cross_build --release -p openshell-server ${EXTRA_CARGO_FEATURES:+--features "$EXTRA_CARGO_FEATURES"} && \ mkdir -p /build/out && \ - cp "$(cross_output_dir release)/openshell-server" /build/out/ + cp "$(cross_output_dir release)/openshell-gateway" /build/out/ FROM rust-deps AS supervisor-workspace ARG OPENSHELL_CARGO_VERSION @@ -190,7 +191,7 @@ RUN useradd --create-home --user-group openshell WORKDIR /app -COPY --from=gateway-builder /build/out/openshell-server /usr/local/bin/ +COPY --from=gateway-builder /build/out/openshell-gateway /usr/local/bin/ RUN mkdir -p /build/crates/openshell-server COPY --chmod=755 crates/openshell-server/migrations /build/crates/openshell-server/migrations @@ -198,9 +199,29 @@ COPY --chmod=755 crates/openshell-server/migrations /build/crates/openshell-serv USER openshell EXPOSE 8080 -ENTRYPOINT ["openshell-server"] +ENTRYPOINT ["openshell-gateway"] CMD ["--port", "8080"] +# --------------------------------------------------------------------------- +# Final supervisor image +# --------------------------------------------------------------------------- +FROM nvcr.io/nvidia/base/ubuntu:noble-20251013 AS supervisor + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates && \ + apt-get install -y --only-upgrade gpgv && \ + rm -rf /var/lib/apt/lists/* + +RUN useradd --create-home --user-group openshell + +WORKDIR /app + +COPY --from=supervisor-builder /build/out/openshell-sandbox /usr/local/bin/ + +USER openshell + +ENTRYPOINT ["openshell-sandbox"] + # --------------------------------------------------------------------------- # Cluster asset stages # --------------------------------------------------------------------------- diff --git a/docs/reference/support-matrix.mdx b/docs/reference/support-matrix.mdx index 744fba53cd..c5eeee5677 100644 --- a/docs/reference/support-matrix.mdx +++ b/docs/reference/support-matrix.mdx @@ -10,7 +10,7 @@ This page lists the platform, software, runtime, and kernel requirements for run ## Supported Platforms -OpenShell publishes multi-architecture container images for `linux/amd64` and `linux/arm64`. The CLI is supported on the following host platforms: +OpenShell publishes multi-architecture container images for `linux/amd64` and `linux/arm64`. The CLI and standalone gateway binary are supported on the following host platforms: | Platform | Architecture | Status | | -------------------------------- | --------------------- | --------- | @@ -19,6 +19,18 @@ OpenShell publishes multi-architecture container images for `linux/amd64` and `l | macOS (Docker Desktop) | Apple Silicon (arm64) | Supported | | Windows (WSL 2 + Docker Desktop) | x86_64 | Experimental | +## Standalone Gateway Binary + +OpenShell publishes standalone `openshell-gateway` release assets for manual download on these platforms: + +| Platform | Artifact pattern | +| --------------------- | ---------------------------------------------- | +| Linux x86_64 (amd64) | `openshell-gateway-x86_64-unknown-linux-gnu` | +| Linux aarch64 (arm64) | `openshell-gateway-aarch64-unknown-linux-gnu` | +| macOS Apple Silicon | `openshell-gateway-aarch64-apple-darwin` | + +These artifacts are attached to GitHub releases. `openshell gateway start` continues to use the published cluster and gateway container images. + ## Software Prerequisites The following software must be installed on the host before using the OpenShell CLI: diff --git a/tasks/docker.toml b/tasks/docker.toml index b4e370ff31..f05c8aab15 100644 --- a/tasks/docker.toml +++ b/tasks/docker.toml @@ -11,29 +11,19 @@ depends = [ ] hide = true -["docker:build"] -description = "Alias for build:docker" -depends = ["build:docker"] -hide = true - ["build:docker:ci"] description = "Build the CI Docker image" run = "tasks/scripts/docker-build-ci.sh" hide = true -["docker:build:ci"] -description = "Alias for build:docker:ci" -depends = ["build:docker:ci"] -hide = true - ["build:docker:gateway"] description = "Build the gateway Docker image" run = "tasks/scripts/docker-build-image.sh gateway" hide = true -["docker:build:gateway"] -description = "Alias for build:docker:gateway" -depends = ["build:docker:gateway"] +["build:docker:supervisor"] +description = "Build the supervisor Docker image" +run = "tasks/scripts/docker-build-image.sh supervisor" hide = true ["build:docker:cluster"] @@ -41,6 +31,11 @@ description = "Build the k3s cluster image (component images pulled at runtime f run = "tasks/scripts/docker-build-image.sh cluster" hide = true +["docker:build:gateway"] +description = "Alias for build:docker:gateway" +depends = ["build:docker:gateway"] +hide = true + ["docker:build:cluster"] description = "Alias for build:docker:cluster" depends = ["build:docker:cluster"] @@ -51,11 +46,6 @@ description = "Build multi-arch cluster image and push to a registry" run = "tasks/scripts/docker-publish-multiarch.sh" hide = true -["docker:build:cluster:multiarch"] -description = "Alias for build:docker:cluster:multiarch" -depends = ["build:docker:cluster:multiarch"] -hide = true - ["docker:cleanup"] description = "Remove stale images, volumes, and build cache not used by the current cluster" run = "scripts/docker-cleanup.sh --force" diff --git a/tasks/scripts/docker-build-image.sh b/tasks/scripts/docker-build-image.sh index 3675d75498..e429d03396 100755 --- a/tasks/scripts/docker-build-image.sh +++ b/tasks/scripts/docker-build-image.sh @@ -38,7 +38,7 @@ detect_rust_scope() { echo "no-rust" } -TARGET=${1:?"Usage: docker-build-image.sh [extra-args...]"} +TARGET=${1:?"Usage: docker-build-image.sh [extra-args...]"} shift DOCKERFILE="deploy/docker/Dockerfile.images" @@ -56,6 +56,11 @@ case "${TARGET}" in IMAGE_NAME="openshell/gateway" DOCKER_TARGET="gateway" ;; + supervisor) + IS_FINAL_IMAGE=1 + IMAGE_NAME="openshell/supervisor" + DOCKER_TARGET="supervisor" + ;; cluster) IS_FINAL_IMAGE=1 IMAGE_NAME="openshell/cluster" From 3bc8e444b3322c4a222a68cab6b9d5713f9711e1 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Thu, 16 Apr 2026 22:00:36 -0700 Subject: [PATCH 005/142] docs(rfc): adopt per-RFC folder structure (#870) Each RFC now lives in its own folder (rfc/NNNN-short-name/) with README.md as the main proposal. This gives each RFC room for diagrams, images, and other supporting files without cluttering the top level. Move 0000-template into the new layout and update rfc/README.md to describe the convention. --- .../README.md} | 0 rfc/README.md | 20 ++++++++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) rename rfc/{0000-template.md => 0000-template/README.md} (100%) diff --git a/rfc/0000-template.md b/rfc/0000-template/README.md similarity index 100% rename from rfc/0000-template.md rename to rfc/0000-template/README.md diff --git a/rfc/README.md b/rfc/README.md index b409cad003..b67cb7c21e 100644 --- a/rfc/README.md +++ b/rfc/README.md @@ -87,11 +87,25 @@ An RFC can be in one of the following states: ### 1. Reserve an RFC number -Look at the existing RFCs in this directory and choose the next available number. If two authors happen to pick the same number on separate branches, the conflict is resolved during PR review — the later PR simply picks the next available number. +Look at the existing RFC folders in this directory and choose the next available number. If two authors happen to pick the same number on separate branches, the conflict is resolved during PR review — the later PR simply picks the next available number. ### 2. Create your RFC -Copy `0000-template.md` to `NNNN-my-feature.md` where `NNNN` is your RFC number (zero-padded to 4 digits) and `my-feature` is a short descriptive name. +Each RFC lives in its own folder: + +``` +rfc/NNNN-my-feature/ + README.md + (optional: diagrams, images, supporting files) +``` + +Where `NNNN` is your RFC number (zero-padded to 4 digits) and `my-feature` is a short descriptive name. The main proposal goes in `README.md` so GitHub renders it when browsing the folder. + +To start a new RFC, copy the template folder: + +```shell +cp -r rfc/0000-template rfc/NNNN-my-feature +``` Fill in the metadata and start writing. The state should be `draft` while you're iterating. @@ -129,7 +143,7 @@ graph LR ``` ```` -Prefer Mermaid over external image files whenever possible. If a diagram is too complex for Mermaid (e.g., detailed UI mockups), commit the image to the same directory as the RFC and reference it with a relative path. +Prefer Mermaid over external image files whenever possible. If a diagram is too complex for Mermaid (e.g., detailed UI mockups), commit the image to the RFC's folder alongside its `README.md` and reference it with a relative path. ## Making changes to an RFC From 4e8dbcfe5c5738edf9658dd9e7f6d090707dc96e Mon Sep 17 00:00:00 2001 From: "John T. Myers" <9696606+johntmyers@users.noreply.github.com> Date: Fri, 17 Apr 2026 08:26:11 -0700 Subject: [PATCH 006/142] fix(sandbox): harden seccomp, inference routing, and process limits (#869) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(sandbox): block AF_NETLINK in seccomp unconditionally Move AF_NETLINK to the unconditional socket-domain block list alongside AF_PACKET, AF_BLUETOOTH, and AF_VSOCK. Previously it was only blocked in NetworkMode::Block, leaving it accessible in Proxy mode where network namespace isolation already scopes netlink to the sandbox's own veth — making this a defense-in-depth hardening rather than a live exposure. Closes OS-94 * fix(sandbox): scope inference.local interception to port 443 The pre-OPA interception for inference.local matched on hostname alone, allowing any port to bypass OPA policy evaluation — including under deny-all (network_policies: {}). Add a port check so only port 443 takes the interception path; all other ports on inference.local now fall through to OPA and are subject to normal policy evaluation. Closes OS-95 * fix(sandbox): enforce RLIMIT_NPROC to prevent fork bomb DoS Set a hard limit of 512 processes per UID in harden_child_process(), applied before privilege drop so the sandbox user cannot raise it. Prevents unrestricted fork() from exhausting the process table — most relevant for local dev mode where K8s pod cgroup pids.max is absent. Closes OS-96 * chore(sandbox): fix rustfmt formatting for seccomp blocked_domains --------- Co-authored-by: John Myers --- crates/openshell-sandbox/src/process.rs | 16 ++++++++++++++++ crates/openshell-sandbox/src/proxy.rs | 3 ++- .../src/sandbox/linux/seccomp.rs | 8 ++++++-- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/crates/openshell-sandbox/src/process.rs b/crates/openshell-sandbox/src/process.rs index 4e92259294..85a57b4e72 100644 --- a/crates/openshell-sandbox/src/process.rs +++ b/crates/openshell-sandbox/src/process.rs @@ -49,6 +49,22 @@ pub(crate) fn harden_child_process() -> Result<()> { )); } + // Limit process creation to prevent fork bombs. 512 processes per UID is + // sufficient for typical agent workloads (shell, compilers, language servers) + // while preventing runaway forking. Set as a hard limit so the sandbox user + // cannot raise it after privilege drop. + let nproc_limit = libc::rlimit { + rlim_cur: 512, + rlim_max: 512, + }; + let rc = unsafe { libc::setrlimit(libc::RLIMIT_NPROC, &nproc_limit) }; + if rc != 0 { + return Err(miette::miette!( + "Failed to set RLIMIT_NPROC: {}", + std::io::Error::last_os_error() + )); + } + #[cfg(target_os = "linux")] { let rc = unsafe { libc::prctl(libc::PR_SET_DUMPABLE, 0, 0, 0, 0) }; diff --git a/crates/openshell-sandbox/src/proxy.rs b/crates/openshell-sandbox/src/proxy.rs index 6f85e848e2..f91f2c551f 100644 --- a/crates/openshell-sandbox/src/proxy.rs +++ b/crates/openshell-sandbox/src/proxy.rs @@ -27,6 +27,7 @@ use tracing::{debug, warn}; const MAX_HEADER_BYTES: usize = 8192; const INFERENCE_LOCAL_HOST: &str = "inference.local"; +const INFERENCE_LOCAL_PORT: u16 = 443; /// Maximum total bytes for a streaming inference response body (32 MiB). const MAX_STREAMING_BODY: usize = 32 * 1024 * 1024; @@ -354,7 +355,7 @@ async fn handle_tcp_connection( let (host, port) = parse_target(target)?; let host_lc = host.to_ascii_lowercase(); - if host_lc == INFERENCE_LOCAL_HOST { + if host_lc == INFERENCE_LOCAL_HOST && port == INFERENCE_LOCAL_PORT { respond(&mut client, b"HTTP/1.1 200 Connection Established\r\n\r\n").await?; let outcome = handle_inference_interception( client, diff --git a/crates/openshell-sandbox/src/sandbox/linux/seccomp.rs b/crates/openshell-sandbox/src/sandbox/linux/seccomp.rs index 854134cbf4..7b6f0f4127 100644 --- a/crates/openshell-sandbox/src/sandbox/linux/seccomp.rs +++ b/crates/openshell-sandbox/src/sandbox/linux/seccomp.rs @@ -112,11 +112,15 @@ fn build_filter_rules(allow_inet: bool) -> Result let mut rules: BTreeMap> = BTreeMap::new(); // --- Socket domain blocks --- - let mut blocked_domains = vec![libc::AF_PACKET, libc::AF_BLUETOOTH, libc::AF_VSOCK]; + let mut blocked_domains = vec![ + libc::AF_PACKET, + libc::AF_BLUETOOTH, + libc::AF_VSOCK, + libc::AF_NETLINK, + ]; if !allow_inet { blocked_domains.push(libc::AF_INET); blocked_domains.push(libc::AF_INET6); - blocked_domains.push(libc::AF_NETLINK); } for domain in blocked_domains { From e4d6f92d9b803719928eca5187b428a7ca55c1a5 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Fri, 17 Apr 2026 08:50:57 -0700 Subject: [PATCH 007/142] feat(vm): add standalone libkrun compute driver (#858) --- .github/workflows/release-vm-dev.yml | 289 +++- AGENTS.md | 2 + Cargo.lock | 23 + architecture/custom-vm-runtime.md | 17 + architecture/gateway.md | 25 +- crates/openshell-core/src/config.rs | 9 +- crates/openshell-driver-vm/Cargo.toml | 41 + crates/openshell-driver-vm/Makefile | 7 + crates/openshell-driver-vm/README.md | 151 ++ crates/openshell-driver-vm/build.rs | 140 ++ crates/openshell-driver-vm/entitlements.plist | 8 + .../scripts/openshell-vm-sandbox-init.sh | 188 +++ crates/openshell-driver-vm/src/driver.rs | 1363 +++++++++++++++++ .../src/embedded_runtime.rs | 181 +++ crates/openshell-driver-vm/src/ffi.rs | 206 +++ crates/openshell-driver-vm/src/lib.rs | 13 + crates/openshell-driver-vm/src/main.rs | 229 +++ crates/openshell-driver-vm/src/rootfs.rs | 234 +++ crates/openshell-driver-vm/src/runtime.rs | 877 +++++++++++ crates/openshell-driver-vm/start.sh | 63 + crates/openshell-server/Cargo.toml | 1 + crates/openshell-server/src/cli.rs | 64 +- crates/openshell-server/src/compute/mod.rs | 197 ++- crates/openshell-server/src/compute/vm.rs | 429 ++++++ crates/openshell-server/src/grpc/sandbox.rs | 27 +- crates/openshell-server/src/lib.rs | 47 +- deploy/docker/Dockerfile.driver-vm-macos | 118 ++ 27 files changed, 4921 insertions(+), 28 deletions(-) create mode 100644 crates/openshell-driver-vm/Cargo.toml create mode 100644 crates/openshell-driver-vm/Makefile create mode 100644 crates/openshell-driver-vm/README.md create mode 100644 crates/openshell-driver-vm/build.rs create mode 100644 crates/openshell-driver-vm/entitlements.plist create mode 100644 crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh create mode 100644 crates/openshell-driver-vm/src/driver.rs create mode 100644 crates/openshell-driver-vm/src/embedded_runtime.rs create mode 100644 crates/openshell-driver-vm/src/ffi.rs create mode 100644 crates/openshell-driver-vm/src/lib.rs create mode 100644 crates/openshell-driver-vm/src/main.rs create mode 100644 crates/openshell-driver-vm/src/rootfs.rs create mode 100644 crates/openshell-driver-vm/src/runtime.rs create mode 100755 crates/openshell-driver-vm/start.sh create mode 100644 crates/openshell-server/src/compute/vm.rs create mode 100644 deploy/docker/Dockerfile.driver-vm-macos diff --git a/.github/workflows/release-vm-dev.yml b/.github/workflows/release-vm-dev.yml index 06ba474a07..b32bfd8de7 100644 --- a/.github/workflows/release-vm-dev.yml +++ b/.github/workflows/release-vm-dev.yml @@ -416,12 +416,253 @@ jobs: path: artifacts/*.tar.gz retention-days: 5 + # --------------------------------------------------------------------------- + # Build openshell-driver-vm binary (Linux — native on each arch) + # --------------------------------------------------------------------------- + build-driver-vm-linux: + name: Build Driver VM (Linux ${{ matrix.arch }}) + needs: [compute-versions, download-kernel-runtime, build-rootfs] + strategy: + matrix: + include: + - arch: arm64 + runner: build-arm64 + target: aarch64-unknown-linux-gnu + platform: linux-aarch64 + guest_arch: aarch64 + - arch: amd64 + runner: build-amd64 + target: x86_64-unknown-linux-gnu + platform: linux-x86_64 + guest_arch: x86_64 + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + container: + image: ghcr.io/nvidia/openshell/ci:latest + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + options: --privileged + env: + MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SCCACHE_MEMCACHED_ENDPOINT: ${{ vars.SCCACHE_MEMCACHED_ENDPOINT }} + OPENSHELL_IMAGE_TAG: dev + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Mark workspace safe for git + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Fetch tags + run: git fetch --tags --force + + - name: Install tools + run: mise install + + - name: Cache Rust target and registry + uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 + with: + shared-key: driver-vm-linux-${{ matrix.arch }} + cache-directories: .cache/sccache + cache-targets: "true" + + - name: Install zstd + run: apt-get update && apt-get install -y --no-install-recommends zstd && rm -rf /var/lib/apt/lists/* + + - name: Download kernel runtime tarball + uses: actions/download-artifact@v4 + with: + name: kernel-runtime-tarballs + path: runtime-download/ + + - name: Download rootfs tarball + uses: actions/download-artifact@v4 + with: + name: rootfs-${{ matrix.arch }} + path: rootfs-download/ + + - name: Stage compressed runtime for embedding + run: | + set -euo pipefail + COMPRESSED_DIR="${PWD}/target/vm-runtime-compressed" + mkdir -p "$COMPRESSED_DIR" + + # Extract kernel runtime tarball and re-compress individual files + EXTRACT_DIR=$(mktemp -d) + zstd -d "runtime-download/vm-runtime-${{ matrix.platform }}.tar.zst" --stdout \ + | tar -xf - -C "$EXTRACT_DIR" + + echo "Extracted runtime files:" + ls -lah "$EXTRACT_DIR" + + for file in "$EXTRACT_DIR"/*; do + [ -f "$file" ] || continue + name=$(basename "$file") + [ "$name" = "provenance.json" ] && continue + zstd -19 -f -q -T0 -o "${COMPRESSED_DIR}/${name}.zst" "$file" + done + + # Copy rootfs tarball (already zstd-compressed) + cp rootfs-download/rootfs.tar.zst "${COMPRESSED_DIR}/rootfs.tar.zst" + + echo "Staged compressed artifacts:" + ls -lah "$COMPRESSED_DIR" + + - name: Scope workspace to driver-vm crates + run: | + set -euo pipefail + sed -i 's|members = \["crates/\*"\]|members = ["crates/openshell-driver-vm", "crates/openshell-core"]|' Cargo.toml + + - name: Patch workspace version + if: needs.compute-versions.outputs.cargo_version != '' + run: | + set -euo pipefail + sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${{ needs.compute-versions.outputs.cargo_version }}"'"/}' Cargo.toml + + - name: Build openshell-driver-vm + run: | + set -euo pipefail + OPENSHELL_VM_RUNTIME_COMPRESSED_DIR="${PWD}/target/vm-runtime-compressed" \ + mise x -- cargo build --release -p openshell-driver-vm + + - name: sccache stats + if: always() + run: mise x -- sccache --show-stats + + - name: Package binary + run: | + set -euo pipefail + mkdir -p artifacts + tar -czf "artifacts/openshell-driver-vm-${{ matrix.target }}.tar.gz" \ + -C target/release openshell-driver-vm + ls -lh artifacts/ + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: driver-vm-linux-${{ matrix.arch }} + path: artifacts/*.tar.gz + retention-days: 5 + + # --------------------------------------------------------------------------- + # Build openshell-driver-vm binary (macOS ARM64 via osxcross) + # --------------------------------------------------------------------------- + build-driver-vm-macos: + name: Build Driver VM (macOS) + needs: [compute-versions, download-kernel-runtime, build-rootfs] + runs-on: build-amd64 + timeout-minutes: 60 + container: + image: ghcr.io/nvidia/openshell/ci:latest + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + options: --privileged + volumes: + - /var/run/docker.sock:/var/run/docker.sock + env: + MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SCCACHE_MEMCACHED_ENDPOINT: ${{ vars.SCCACHE_MEMCACHED_ENDPOINT }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Mark workspace safe for git + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Fetch tags + run: git fetch --tags --force + + - name: Log in to GHCR + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin + + - name: Set up Docker Buildx + uses: ./.github/actions/setup-buildx + + - name: Install zstd + run: apt-get update && apt-get install -y --no-install-recommends zstd && rm -rf /var/lib/apt/lists/* + + - name: Download kernel runtime tarball + uses: actions/download-artifact@v4 + with: + name: kernel-runtime-tarballs + path: runtime-download/ + + - name: Download rootfs tarball (arm64) + uses: actions/download-artifact@v4 + with: + name: rootfs-arm64 + path: rootfs-download/ + + - name: Prepare compressed runtime directory + run: | + set -euo pipefail + COMPRESSED_DIR="${PWD}/target/vm-runtime-compressed-macos" + mkdir -p "$COMPRESSED_DIR" + + # Extract the darwin runtime tarball and re-compress for embedding. + # The macOS embedded.rs expects: libkrun.dylib.zst, libkrunfw.5.dylib.zst, gvproxy.zst + EXTRACT_DIR=$(mktemp -d) + zstd -d "runtime-download/vm-runtime-darwin-aarch64.tar.zst" --stdout \ + | tar -xf - -C "$EXTRACT_DIR" + + echo "Extracted darwin runtime files:" + ls -lah "$EXTRACT_DIR" + + for file in "$EXTRACT_DIR"/*; do + [ -f "$file" ] || continue + name=$(basename "$file") + [ "$name" = "provenance.json" ] && continue + zstd -19 -f -q -T0 -o "${COMPRESSED_DIR}/${name}.zst" "$file" + done + + # The macOS VM guest is always Linux ARM64, so use the arm64 rootfs + cp rootfs-download/rootfs.tar.zst "${COMPRESSED_DIR}/rootfs.tar.zst" + + echo "Staged macOS compressed artifacts:" + ls -lah "$COMPRESSED_DIR" + + - name: Build macOS binary via Docker (osxcross) + run: | + set -euo pipefail + docker buildx build \ + --file deploy/docker/Dockerfile.driver-vm-macos \ + --build-arg OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" \ + --build-arg OPENSHELL_IMAGE_TAG=dev \ + --build-arg CARGO_TARGET_CACHE_SCOPE="${{ github.sha }}" \ + --build-context vm-runtime-compressed="${PWD}/target/vm-runtime-compressed-macos" \ + --target binary \ + --output type=local,dest=out/ \ + . + + - name: Package binary + run: | + set -euo pipefail + mkdir -p artifacts + tar -czf artifacts/openshell-driver-vm-aarch64-apple-darwin.tar.gz \ + -C out openshell-driver-vm + ls -lh artifacts/ + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: driver-vm-macos + path: artifacts/*.tar.gz + retention-days: 5 + # --------------------------------------------------------------------------- # Upload all VM binaries to the vm-dev rolling release # --------------------------------------------------------------------------- release-vm-dev: name: Release VM Dev - needs: [build-vm-linux, build-vm-macos] + needs: + - build-vm-linux + - build-vm-macos + - build-driver-vm-linux + - build-driver-vm-macos runs-on: build-amd64 timeout-minutes: 10 steps: @@ -430,7 +671,7 @@ jobs: - name: Download all VM binary artifacts uses: actions/download-artifact@v4 with: - pattern: vm-* + pattern: "{vm-*,driver-vm-*}" path: release/ merge-multiple: true @@ -438,21 +679,30 @@ jobs: run: | set -euo pipefail mkdir -p release-final - # Only include the openshell-vm binary tarballs, not kernel runtime - cp release/openshell-vm-*.tar.gz release-final/ - count=$(ls release-final/openshell-vm-*.tar.gz 2>/dev/null | wc -l) - if [ "$count" -eq 0 ]; then - echo "ERROR: No VM binary tarballs found in release/" >&2 + # Include both openshell-vm and openshell-driver-vm binary tarballs. + # Exclude kernel runtime tarballs (they come from release-vm-kernel.yml). + for pattern in 'openshell-vm-*.tar.gz' 'openshell-driver-vm-*.tar.gz'; do + for file in release/${pattern}; do + [ -f "$file" ] || continue + cp "$file" release-final/ + done + done + vm_count=$(ls release-final/openshell-vm-*.tar.gz 2>/dev/null | wc -l) + driver_count=$(ls release-final/openshell-driver-vm-*.tar.gz 2>/dev/null | wc -l) + if [ "$vm_count" -eq 0 ] || [ "$driver_count" -eq 0 ]; then + echo "ERROR: Missing binary tarballs (openshell-vm=${vm_count}, openshell-driver-vm=${driver_count})" >&2 + ls -la release/ || true exit 1 fi - echo "Release artifacts (${count} binaries):" + echo "Release artifacts (openshell-vm=${vm_count}, openshell-driver-vm=${driver_count}):" ls -lh release-final/ - name: Generate checksums run: | set -euo pipefail cd release-final - sha256sum openshell-vm-*.tar.gz > vm-binary-checksums-sha256.txt + sha256sum openshell-vm-*.tar.gz openshell-driver-vm-*.tar.gz \ + > vm-binary-checksums-sha256.txt cat vm-binary-checksums-sha256.txt - name: Ensure vm-dev tag exists @@ -479,7 +729,11 @@ jobs: } // Delete old VM binary assets (keep kernel runtime assets) for (const asset of release.data.assets) { - if (asset.name.startsWith('openshell-vm-') || asset.name === 'vm-binary-checksums-sha256.txt') { + if ( + asset.name.startsWith('openshell-vm-') || + asset.name.startsWith('openshell-driver-vm-') || + asset.name === 'vm-binary-checksums-sha256.txt' + ) { core.info(`Deleting stale asset: ${asset.name}`); await github.rest.repos.deleteReleaseAsset({ owner, repo, asset_id: asset.id }); } @@ -520,6 +774,18 @@ jobs: | Linux x86_64 | `openshell-vm-x86_64-unknown-linux-gnu.tar.gz` | | macOS ARM64 | `openshell-vm-aarch64-apple-darwin.tar.gz` | + ### VM Compute Driver Binaries + + `openshell-driver-vm` binaries with embedded kernel runtime and sandbox rootfs. + Launched by the gateway when `--drivers=vm` is configured. Rebuilt on every + push to main alongside the openshell-vm binaries. + + | Platform | Artifact | + |----------|----------| + | Linux ARM64 | `openshell-driver-vm-aarch64-unknown-linux-gnu.tar.gz` | + | Linux x86_64 | `openshell-driver-vm-x86_64-unknown-linux-gnu.tar.gz` | + | macOS ARM64 | `openshell-driver-vm-aarch64-apple-darwin.tar.gz` | + ### Quick install ``` @@ -532,4 +798,7 @@ jobs: release-final/openshell-vm-aarch64-unknown-linux-gnu.tar.gz release-final/openshell-vm-x86_64-unknown-linux-gnu.tar.gz release-final/openshell-vm-aarch64-apple-darwin.tar.gz + release-final/openshell-driver-vm-aarch64-unknown-linux-gnu.tar.gz + release-final/openshell-driver-vm-x86_64-unknown-linux-gnu.tar.gz + release-final/openshell-driver-vm-aarch64-apple-darwin.tar.gz release-final/vm-binary-checksums-sha256.txt diff --git a/AGENTS.md b/AGENTS.md index a6cbd29ce8..8968c52960 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -40,6 +40,8 @@ These pipelines connect skills into end-to-end workflows. Individual skill files | `crates/openshell-providers/` | Provider management | Credential provider backends | | `crates/openshell-tui/` | Terminal UI | Ratatui-based dashboard for monitoring | | `crates/openshell-vm/` | MicroVM runtime | Experimental, work-in-progress libkrun-based VM execution | +| `crates/openshell-driver-kubernetes/` | Kubernetes compute driver | In-process `ComputeDriver` backend for K8s sandbox pods | +| `crates/openshell-driver-vm/` | VM compute driver | Standalone libkrun-backed `ComputeDriver` subprocess (embeds its own rootfs + runtime) | | `python/openshell/` | Python SDK | Python bindings and CLI packaging | | `proto/` | Protobuf definitions | gRPC service contracts | | `deploy/` | Docker, Helm, K8s | Dockerfiles, Helm chart, manifests | diff --git a/Cargo.lock b/Cargo.lock index e4057f75cc..4b29a0c7f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3090,6 +3090,28 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "openshell-driver-vm" +version = "0.0.0" +dependencies = [ + "clap", + "futures", + "libc", + "libloading", + "miette", + "nix", + "openshell-core", + "prost-types", + "tar", + "tokio", + "tokio-stream", + "tonic", + "tracing", + "tracing-subscriber", + "url", + "zstd", +] + [[package]] name = "openshell-ocsf" version = "0.0.0" @@ -3245,6 +3267,7 @@ dependencies = [ "tower-http 0.6.8", "tracing", "tracing-subscriber", + "url", "uuid", "wiremock", ] diff --git a/architecture/custom-vm-runtime.md b/architecture/custom-vm-runtime.md index ce4d0bf393..548b86d178 100644 --- a/architecture/custom-vm-runtime.md +++ b/architecture/custom-vm-runtime.md @@ -80,6 +80,23 @@ The embedded rootfs uses a "minimal" configuration: Container images are pulled on demand when sandboxes are created. First boot takes ~30-60s as k3s initializes; subsequent boots use cached state for ~3-5s startup. +For the VM compute driver, the same embedded rootfs is rewritten into a +supervisor-only sandbox guest before boot: + +- removes k3s state and Kubernetes manifests from the extracted rootfs +- installs `/srv/openshell-vm-sandbox-init.sh` +- boots directly into `openshell-sandbox` instead of `openshell-vm-init.sh` +- keeps the same embedded libkrun/libkrunfw kernel/runtime bundle + +`openshell-driver-vm` now embeds the sandbox rootfs tarball independently so it can +prepare sandbox guests without linking against the `openshell-vm` Rust crate. +It now also embeds the minimal libkrun/libkrunfw bundle it needs for sandbox +boots and launches sandbox guests via a hidden helper mode in the +`openshell-driver-vm` binary itself, without depending on the `openshell-vm` +binary. The helper still starts its own embedded `gvproxy` instance to provide +virtio-net guest egress plus the single inbound SSH port forward used by the +compute driver. + For fully air-gapped environments requiring pre-loaded images, build with: ```bash mise run vm:rootfs # Full rootfs (~2GB, includes images) diff --git a/architecture/gateway.md b/architecture/gateway.md index 9677dfc47e..02f487050d 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -72,6 +72,7 @@ graph TD | Persistence: Postgres | `crates/openshell-server/src/persistence/postgres.rs` | `PostgresStore` with sqlx | | Compute runtime | `crates/openshell-server/src/compute/mod.rs` | `ComputeRuntime`, gateway-owned sandbox lifecycle orchestration over a compute backend | | Compute driver: Kubernetes | `crates/openshell-driver-kubernetes/src/driver.rs` | Kubernetes CRD create/delete, endpoint resolution, watch stream, pod template translation | +| Compute driver: VM | `crates/openshell-driver-vm/src/driver.rs` | Per-sandbox microVM create/delete, localhost endpoint resolution, watch stream, supervisor-only guest boot | | Sandbox index | `crates/openshell-server/src/sandbox_index.rs` | `SandboxIndex` -- in-memory name/pod-to-id correlation | | Watch bus | `crates/openshell-server/src/sandbox_watch.rs` | `SandboxWatchBus` -- in-memory broadcast for persisted sandbox updates | | Tracing bus | `crates/openshell-server/src/tracing_bus.rs` | `TracingLogBus` -- captures tracing events keyed by `sandbox_id` | @@ -96,7 +97,9 @@ The gateway boots in `main()` (`crates/openshell-server/src/main.rs`) and procee 4. **Build `Config`** -- Assembles a `openshell_core::Config` from the parsed arguments. 5. **Call `run_server()`** (`crates/openshell-server/src/lib.rs`): 1. Connect to the persistence store (`Store::connect`), which auto-detects SQLite vs Postgres from the URL prefix and runs migrations. - 2. Create `ComputeRuntime` with an in-process `ComputeDriverService` backed by `KubernetesComputeDriver`, so the gateway calls the `openshell.compute.v1.ComputeDriver` RPC surface even without transport. + 2. Create `ComputeRuntime` with a `ComputeDriver` implementation selected by `OPENSHELL_DRIVERS`: + - `kubernetes` wraps `KubernetesComputeDriver` in `ComputeDriverService`, so the gateway uses the `openshell.compute.v1.ComputeDriver` RPC surface even without transport. + - `vm` spawns the sibling `openshell-driver-vm` binary as a local compute-driver process, connects to it over a Unix domain socket, and keeps the libkrun/rootfs runtime out of the gateway binary. 3. Build `ServerState` (shared via `Arc` across all handlers). 4. **Spawn background tasks**: - `ComputeRuntime::spawn_watchers` -- consumes the compute-driver watch stream, republishes platform events, and runs a periodic `ListSandboxes` snapshot reconcile so the store-backed public sandbox reads stay aligned with the compute driver. @@ -123,6 +126,15 @@ All configuration is via CLI flags with environment variable fallbacks. The `--d | `--sandbox-namespace` | `OPENSHELL_SANDBOX_NAMESPACE` | `default` | Kubernetes namespace for sandbox CRDs | | `--sandbox-image` | `OPENSHELL_SANDBOX_IMAGE` | None | Default container image for sandbox pods | | `--grpc-endpoint` | `OPENSHELL_GRPC_ENDPOINT` | None | gRPC endpoint reachable from within the cluster (for sandbox callbacks) | +| `--drivers` | `OPENSHELL_DRIVERS` | `kubernetes` | Compute backend to use. Current options are `kubernetes` and `vm`. | +| `--vm-driver-state-dir` | `OPENSHELL_VM_DRIVER_STATE_DIR` | `target/openshell-vm-driver` | Host directory for VM sandbox rootfs, console logs, and runtime state | +| `--vm-compute-driver-bin` | `OPENSHELL_VM_COMPUTE_DRIVER_BIN` | sibling `openshell-driver-vm` binary | Local VM compute-driver process spawned by the gateway | +| `--vm-krun-log-level` | `OPENSHELL_VM_KRUN_LOG_LEVEL` | `1` | libkrun log level for VM helper processes | +| `--vm-driver-vcpus` | `OPENSHELL_VM_DRIVER_VCPUS` | `2` | Default vCPU count for VM sandboxes | +| `--vm-driver-mem-mib` | `OPENSHELL_VM_DRIVER_MEM_MIB` | `2048` | Default memory allocation for VM sandboxes in MiB | +| `--vm-tls-ca` | `OPENSHELL_VM_TLS_CA` | None | CA cert copied into VM guests for gateway mTLS | +| `--vm-tls-cert` | `OPENSHELL_VM_TLS_CERT` | None | Client cert copied into VM guests for gateway mTLS | +| `--vm-tls-key` | `OPENSHELL_VM_TLS_KEY` | None | Client private key copied into VM guests for gateway mTLS | | `--ssh-gateway-host` | `OPENSHELL_SSH_GATEWAY_HOST` | `127.0.0.1` | Public hostname returned in SSH session responses | | `--ssh-gateway-port` | `OPENSHELL_SSH_GATEWAY_PORT` | `8080` | Public port returned in SSH session responses | | `--ssh-connect-path` | `OPENSHELL_SSH_CONNECT_PATH` | `/connect/ssh` | HTTP path for SSH CONNECT/upgrade | @@ -544,6 +556,17 @@ The Kubernetes driver also watches namespace-scoped Kubernetes `Event` objects a Matched events are published to the `PlatformEventBus` as `SandboxStreamEvent::Event` payloads. +## VM Driver + +`VmDriver` (`crates/openshell-driver-vm/src/driver.rs`) is served by the standalone `openshell-driver-vm` process. The gateway spawns that binary on demand, talks to it over the internal `openshell.compute.v1.ComputeDriver` gRPC contract via a Unix domain socket, and keeps VM runtime dependencies out of `openshell-server`. + +- **Create**: The VM driver process allocates a localhost SSH port, prepares a sandbox-specific rootfs from its own embedded `rootfs.tar.zst`, injects an explicitly configured guest mTLS bundle when the gateway callback endpoint is `https://`, then re-execs itself in a hidden helper mode that loads libkrun directly and boots `/srv/openshell-vm-sandbox-init.sh`. +- **Networking**: The helper starts an embedded `gvproxy`, wires it into libkrun as virtio-net, and exposes the single inbound SSH port (`host_port:2222`) through gvproxy’s forwarder API. This keeps VM launch inside `openshell-driver-vm` without depending on the `openshell-vm` binary. +- **Gateway callback**: The guest init script configures `eth0` for gvproxy networking, prefers the configured `OPENSHELL_GRPC_ENDPOINT`, and falls back to host aliases or the gvproxy gateway IP (`192.168.127.1`) when local hostname resolution is unavailable on macOS. +- **Guest boot**: The sandbox guest runs a minimal init script that skips k3s and starts `openshell-sandbox` directly as PID 1 inside the VM. +- **Endpoint resolution**: Returns `127.0.0.1:` for SSH/exec transport. +- **Watch stream**: Emits provisioning, ready, error, deleting, deleted, and platform-event updates so the gateway store remains the durable source of truth. + ## Sandbox Index `SandboxIndex` (`crates/openshell-server/src/sandbox_index.rs`) maintains two in-memory maps protected by an `RwLock`: diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index 279752b4fd..01b5c23720 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -14,6 +14,7 @@ use std::str::FromStr; #[serde(rename_all = "snake_case")] pub enum ComputeDriverKind { Kubernetes, + Vm, Podman, } @@ -22,6 +23,7 @@ impl ComputeDriverKind { pub const fn as_str(self) -> &'static str { match self { Self::Kubernetes => "kubernetes", + Self::Vm => "vm", Self::Podman => "podman", } } @@ -39,9 +41,10 @@ impl FromStr for ComputeDriverKind { fn from_str(value: &str) -> Result { match value.trim().to_ascii_lowercase().as_str() { "kubernetes" => Ok(Self::Kubernetes), + "vm" => Ok(Self::Vm), "podman" => Ok(Self::Podman), other => Err(format!( - "unsupported compute driver '{other}'. expected one of: kubernetes, podman" + "unsupported compute driver '{other}'. expected one of: kubernetes, vm, podman" )), } } @@ -358,6 +361,10 @@ mod tests { "kubernetes".parse::().unwrap(), ComputeDriverKind::Kubernetes ); + assert_eq!( + "vm".parse::().unwrap(), + ComputeDriverKind::Vm + ); assert_eq!( "podman".parse::().unwrap(), ComputeDriverKind::Podman diff --git a/crates/openshell-driver-vm/Cargo.toml b/crates/openshell-driver-vm/Cargo.toml new file mode 100644 index 0000000000..368716ef96 --- /dev/null +++ b/crates/openshell-driver-vm/Cargo.toml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "openshell-driver-vm" +description = "MicroVM compute driver for OpenShell" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[lib] +name = "openshell_driver_vm" +path = "src/lib.rs" + +[[bin]] +name = "openshell-driver-vm" +path = "src/main.rs" + +[dependencies] +openshell-core = { path = "../openshell-core" } + +tokio = { workspace = true } +tonic = { workspace = true, features = ["transport"] } +prost-types = { workspace = true } +futures = { workspace = true } +tokio-stream = { workspace = true, features = ["net"] } +nix = { workspace = true } +clap = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +miette = { workspace = true } +url = { workspace = true } +libc = "0.2" +libloading = "0.8" +tar = "0.4" +zstd = "0.13" + +[lints] +workspace = true diff --git a/crates/openshell-driver-vm/Makefile b/crates/openshell-driver-vm/Makefile new file mode 100644 index 0000000000..e1c360f3da --- /dev/null +++ b/crates/openshell-driver-vm/Makefile @@ -0,0 +1,7 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +.PHONY: start + +start: + ./start.sh diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md new file mode 100644 index 0000000000..6b874eb105 --- /dev/null +++ b/crates/openshell-driver-vm/README.md @@ -0,0 +1,151 @@ +# openshell-driver-vm + +> Status: Experimental. The VM compute driver is under active development and the interface still has VM-specific plumbing that will be generalized. + +Standalone libkrun-backed [`ComputeDriver`](../../proto/compute_driver.proto) for OpenShell. The gateway spawns this binary as a subprocess, talks to it over a Unix domain socket with the `openshell.compute.v1.ComputeDriver` gRPC surface, and lets it manage per-sandbox microVMs. The runtime (libkrun + libkrunfw + gvproxy) and sandbox rootfs are embedded directly in the binary — no sibling files required at runtime. + +## How it fits together + +```mermaid +flowchart LR + subgraph host["Host process"] + gateway["openshell-server
(compute::vm::spawn)"] + driver["openshell-driver-vm
├── libkrun (VM)
├── gvproxy (net)
└── rootfs.tar.zst"] + gateway <-->|"gRPC over UDS
compute-driver.sock"| driver + end + + subgraph guest["Per-sandbox microVM"] + init["/srv/openshell-vm-
sandbox-init.sh"] + supervisor["/opt/openshell/bin/
openshell-sandbox
(PID 1)"] + init --> supervisor + end + + driver -->|"CreateSandbox
boots via libkrun"| guest + supervisor -.->|"gRPC callback
--grpc-endpoint"| gateway + + client["openshell-cli"] -->|"SSH proxy
127.0.0.1:<port>"| supervisor + client -->|"CreateSandbox / Watch"| gateway +``` + +Sandbox guests execute `/opt/openshell/bin/openshell-sandbox` as PID 1 inside the VM. gvproxy exposes a single inbound SSH port (`host:` → `guest:2222`) and provides virtio-net egress. + +## Quick start (recommended) + +`start.sh` handles runtime setup, builds, codesigning, and environment wiring. From the repo root: + +```shell +crates/openshell-driver-vm/start.sh +``` + +or equivalently: + +```shell +make -C crates/openshell-driver-vm start +``` + +First run takes a few minutes while `mise run vm:setup` stages libkrun/libkrunfw/gvproxy and `mise run vm:rootfs -- --base` builds the embedded rootfs. Subsequent runs are cached. State lives under `target/openshell-vm-driver-dev/` (SQLite DB + per-sandbox rootfs + `compute-driver.sock`). + +Override via environment: + +```shell +OPENSHELL_SERVER_PORT=9090 \ +OPENSHELL_SSH_HANDSHAKE_SECRET=$(openssl rand -hex 32) \ +crates/openshell-driver-vm/start.sh +``` + +Teardown: + +```shell +rm -rf target/openshell-vm-driver-dev +``` + +## Manual equivalent + +If you want to drive the launch yourself instead of using `start.sh`: + +```shell +# 1. Stage runtime artifacts + base rootfs into target/vm-runtime-compressed/ +mise run vm:setup +mise run vm:rootfs -- --base # if rootfs.tar.zst is not already present + +# 2. Build both binaries with the staged artifacts embedded +OPENSHELL_VM_RUNTIME_COMPRESSED_DIR=$PWD/target/vm-runtime-compressed \ + cargo build -p openshell-server -p openshell-driver-vm + +# 3. macOS only: codesign the driver for Hypervisor.framework +codesign \ + --entitlements crates/openshell-driver-vm/entitlements.plist \ + --force -s - target/debug/openshell-driver-vm + +# 4. Start the gateway with the VM driver +mkdir -p target/openshell-vm-driver-dev +target/debug/openshell-gateway \ + --drivers vm \ + --disable-tls \ + --database-url sqlite:target/openshell-vm-driver-dev/openshell.db \ + --grpc-endpoint http://host.containers.internal:8080 \ + --ssh-handshake-secret dev-vm-driver-secret \ + --ssh-gateway-host 127.0.0.1 \ + --ssh-gateway-port 8080 \ + --vm-driver-state-dir $PWD/target/openshell-vm-driver-dev +``` + +The gateway discovers `openshell-driver-vm` as a sibling of its own binary. Pass `--vm-compute-driver-bin /path/to/openshell-driver-vm` (or set `OPENSHELL_VM_COMPUTE_DRIVER_BIN`) to override. + +## Flags + +| Flag | Env var | Default | Purpose | +|---|---|---|---| +| `--drivers vm` | `OPENSHELL_DRIVERS` | `kubernetes` | Select the VM compute driver. | +| `--grpc-endpoint URL` | `OPENSHELL_GRPC_ENDPOINT` | — | Required. URL the sandbox guest calls back to. Use a host alias that resolves to the gateway's host from inside the VM (gvproxy answers `host.containers.internal` and `host.openshell.internal` to `192.168.127.1`). | +| `--vm-driver-state-dir DIR` | `OPENSHELL_VM_DRIVER_STATE_DIR` | `target/openshell-vm-driver` | Per-sandbox rootfs, console logs, and the `compute-driver.sock` UDS. | +| `--vm-compute-driver-bin PATH` | `OPENSHELL_VM_COMPUTE_DRIVER_BIN` | sibling of gateway binary | Override the driver binary path. | +| `--vm-driver-vcpus N` | `OPENSHELL_VM_DRIVER_VCPUS` | `2` | vCPUs per sandbox. | +| `--vm-driver-mem-mib N` | `OPENSHELL_VM_DRIVER_MEM_MIB` | `2048` | Memory per sandbox, in MiB. | +| `--vm-krun-log-level N` | `OPENSHELL_VM_KRUN_LOG_LEVEL` | `1` | libkrun verbosity (0–5). | +| `--vm-tls-ca PATH` | `OPENSHELL_VM_TLS_CA` | — | CA cert for the guest's mTLS client bundle. Required when `--grpc-endpoint` uses `https://`. | +| `--vm-tls-cert PATH` | `OPENSHELL_VM_TLS_CERT` | — | Guest client certificate. | +| `--vm-tls-key PATH` | `OPENSHELL_VM_TLS_KEY` | — | Guest client private key. | + +See [`openshell-gateway --help`](../openshell-server/src/cli.rs) for the full flag surface shared with the Kubernetes driver. + +## Verifying the gateway + +In another terminal: + +```shell +export OPENSHELL_GATEWAY_URL=http://127.0.0.1:8080 +cargo run -p openshell-cli -- gateway register local --url $OPENSHELL_GATEWAY_URL --no-tls +cargo run -p openshell-cli -- sandbox create --name demo +cargo run -p openshell-cli -- sandbox connect demo +``` + +First sandbox takes 10–30 seconds to boot (rootfs extraction + libkrun + guest init). Subsequent creates reuse the prepared sandbox rootfs. + +## Logs and debugging + +Raise log verbosity for both processes: + +```shell +RUST_LOG=openshell_server=debug,openshell_driver_vm=debug \ + crates/openshell-driver-vm/start.sh +``` + +The VM guest's serial console is appended to `//console.log`. The `compute-driver.sock` lives at `/compute-driver.sock`; the gateway removes it on clean shutdown via `ManagedDriverProcess::drop`. + +## Prerequisites + +- macOS on Apple Silicon, or Linux on aarch64/x86_64 with KVM +- Rust toolchain +- [mise](https://mise.jdx.dev/) task runner +- Docker (needed by `mise run vm:rootfs` to build the base rootfs) +- `gh` CLI (used by `mise run vm:setup` to download pre-built runtime artifacts) + +## Relationship to `openshell-vm` + +`openshell-vm` is a separate, legacy crate that runs the **whole OpenShell gateway inside a single VM**. `openshell-driver-vm` is the compute driver called by a host-resident gateway to spawn **per-sandbox VMs**. Both embed libkrun but share no Rust code — the driver vendors its own rootfs handling and runtime loader so `openshell-server` never has to link libkrun. + +## TODOs + +- The gateway still configures the driver via CLI args; this will move to a gRPC bootstrap call so the driver interface is uniform across backends. See the `TODO(driver-abstraction)` notes in `crates/openshell-server/src/lib.rs` and `crates/openshell-server/src/compute/vm.rs`. +- macOS codesigning is handled by `start.sh`; a packaged release would need signing in CI. diff --git a/crates/openshell-driver-vm/build.rs b/crates/openshell-driver-vm/build.rs new file mode 100644 index 0000000000..174a90fc87 --- /dev/null +++ b/crates/openshell-driver-vm/build.rs @@ -0,0 +1,140 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Build script for openshell-driver-vm. +//! +//! This crate embeds the sandbox rootfs plus the minimal libkrun runtime +//! artifacts it needs to boot base VMs without depending on the openshell-vm +//! binary or crate. + +use std::path::PathBuf; +use std::{env, fs}; + +fn main() { + println!("cargo:rerun-if-env-changed=OPENSHELL_VM_RUNTIME_COMPRESSED_DIR"); + + if let Ok(dir) = env::var("OPENSHELL_VM_RUNTIME_COMPRESSED_DIR") { + println!("cargo:rerun-if-changed={dir}"); + for name in &[ + "libkrun.so.zst", + "libkrunfw.so.5.zst", + "libkrun.dylib.zst", + "libkrunfw.5.dylib.zst", + "gvproxy.zst", + "rootfs.tar.zst", + ] { + println!("cargo:rerun-if-changed={dir}/{name}"); + } + } + + let out_dir = PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR not set")); + let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap_or_default(); + let target_arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap_or_default(); + + let (libkrun_name, libkrunfw_name) = match target_os.as_str() { + "macos" => ("libkrun.dylib", "libkrunfw.5.dylib"), + "linux" => ("libkrun.so", "libkrunfw.so.5"), + _ => { + println!("cargo:warning=VM runtime not available for {target_os}-{target_arch}"); + generate_stub_resources(&out_dir, &["libkrun", "libkrunfw", "rootfs.tar.zst"]); + return; + } + }; + + let compressed_dir = if let Ok(dir) = env::var("OPENSHELL_VM_RUNTIME_COMPRESSED_DIR") { + PathBuf::from(dir) + } else { + println!("cargo:warning=OPENSHELL_VM_RUNTIME_COMPRESSED_DIR not set"); + println!("cargo:warning=Run: mise run vm:setup"); + generate_stub_resources( + &out_dir, + &[ + &format!("{libkrun_name}.zst"), + &format!("{libkrunfw_name}.zst"), + "gvproxy.zst", + "rootfs.tar.zst", + ], + ); + return; + }; + + if !compressed_dir.is_dir() { + println!( + "cargo:warning=Compressed runtime dir not found: {}", + compressed_dir.display() + ); + println!("cargo:warning=Run: mise run vm:setup"); + generate_stub_resources( + &out_dir, + &[ + &format!("{libkrun_name}.zst"), + &format!("{libkrunfw_name}.zst"), + "gvproxy.zst", + "rootfs.tar.zst", + ], + ); + return; + } + + let files = [ + (format!("{libkrun_name}.zst"), format!("{libkrun_name}.zst")), + ( + format!("{libkrunfw_name}.zst"), + format!("{libkrunfw_name}.zst"), + ), + ("gvproxy.zst".to_string(), "gvproxy.zst".to_string()), + ("rootfs.tar.zst".to_string(), "rootfs.tar.zst".to_string()), + ]; + + let mut all_found = true; + for (src_name, dst_name) in &files { + let src_path = compressed_dir.join(src_name); + let dst_path = out_dir.join(dst_name); + + if !src_path.exists() { + println!( + "cargo:warning=Missing compressed artifact: {}", + src_path.display() + ); + all_found = false; + continue; + } + + if dst_path.exists() { + let _ = fs::remove_file(&dst_path); + } + fs::copy(&src_path, &dst_path).unwrap_or_else(|e| { + panic!( + "Failed to copy {} to {}: {}", + src_path.display(), + dst_path.display(), + e + ) + }); + let size = fs::metadata(&dst_path).map(|m| m.len()).unwrap_or(0); + println!("cargo:warning=Embedded {src_name}: {size} bytes"); + } + + if !all_found { + println!("cargo:warning=Some artifacts missing. Run: mise run vm:setup"); + generate_stub_resources( + &out_dir, + &[ + &format!("{libkrun_name}.zst"), + &format!("{libkrunfw_name}.zst"), + "gvproxy.zst", + "rootfs.tar.zst", + ], + ); + } +} + +fn generate_stub_resources(out_dir: &PathBuf, names: &[&str]) { + for name in names { + let path = out_dir.join(name); + if !path.exists() { + fs::write(&path, b"") + .unwrap_or_else(|e| panic!("Failed to write stub {}: {}", path.display(), e)); + } + } +} diff --git a/crates/openshell-driver-vm/entitlements.plist b/crates/openshell-driver-vm/entitlements.plist new file mode 100644 index 0000000000..154f3308ef --- /dev/null +++ b/crates/openshell-driver-vm/entitlements.plist @@ -0,0 +1,8 @@ + + + + + com.apple.security.hypervisor + + + diff --git a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh new file mode 100644 index 0000000000..70dda5acbc --- /dev/null +++ b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh @@ -0,0 +1,188 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Minimal init for sandbox VMs. Runs as PID 1 inside the guest, mounts the +# essential filesystems, configures gvproxy networking when present, then +# execs the OpenShell sandbox supervisor. + +set -euo pipefail + +BOOT_START=$(date +%s%3N 2>/dev/null || date +%s) + +ts() { + local now + now=$(date +%s%3N 2>/dev/null || date +%s) + local elapsed=$((now - BOOT_START)) + printf "[%d.%03ds] %s\n" $((elapsed / 1000)) $((elapsed % 1000)) "$*" +} + +parse_endpoint() { + local endpoint="$1" + local scheme rest authority path host port + + case "$endpoint" in + *://*) + scheme="${endpoint%%://*}" + rest="${endpoint#*://}" + ;; + *) + return 1 + ;; + esac + + authority="${rest%%/*}" + path="${rest#"$authority"}" + if [ "$path" = "$rest" ]; then + path="" + fi + + if [[ "$authority" =~ ^\[([^]]+)\]:(.+)$ ]]; then + host="${BASH_REMATCH[1]}" + port="${BASH_REMATCH[2]}" + elif [[ "$authority" =~ ^\[([^]]+)\]$ ]]; then + host="${BASH_REMATCH[1]}" + port="" + elif [[ "$authority" == *:* ]]; then + host="${authority%%:*}" + port="${authority##*:}" + else + host="$authority" + port="" + fi + + if [ -z "$port" ]; then + case "$scheme" in + https) port="443" ;; + *) port="80" ;; + esac + fi + + printf '%s\n%s\n%s\n%s\n' "$scheme" "$host" "$port" "$path" +} + +tcp_probe() { + local host="$1" + local port="$2" + + if command -v timeout >/dev/null 2>&1; then + timeout 2 bash -c "exec 3<>/dev/tcp/${host}/${port}" >/dev/null 2>&1 + else + bash -c "exec 3<>/dev/tcp/${host}/${port}" >/dev/null 2>&1 + fi +} + +rewrite_openshell_endpoint_if_needed() { + local endpoint="${OPENSHELL_ENDPOINT:-}" + [ -n "$endpoint" ] || return 0 + + local parsed + if ! parsed="$(parse_endpoint "$endpoint")"; then + ts "WARNING: could not parse OPENSHELL_ENDPOINT=$endpoint" + return 0 + fi + + local scheme host port path + scheme="$(printf '%s\n' "$parsed" | sed -n '1p')" + host="$(printf '%s\n' "$parsed" | sed -n '2p')" + port="$(printf '%s\n' "$parsed" | sed -n '3p')" + path="$(printf '%s\n' "$parsed" | sed -n '4p')" + + if tcp_probe "$host" "$port"; then + return 0 + fi + + for candidate in host.containers.internal host.docker.internal 192.168.127.1; do + if [ "$candidate" = "$host" ]; then + continue + fi + if tcp_probe "$candidate" "$port"; then + local authority="$candidate" + if ! { [ "$scheme" = "http" ] && [ "$port" = "80" ]; } \ + && ! { [ "$scheme" = "https" ] && [ "$port" = "443" ]; }; then + authority="${authority}:${port}" + fi + export OPENSHELL_ENDPOINT="${scheme}://${authority}${path}" + ts "rewrote OPENSHELL_ENDPOINT to ${OPENSHELL_ENDPOINT}" + return 0 + fi + done + + ts "WARNING: could not reach OpenShell endpoint ${host}:${port}" +} + +mount -t proc proc /proc 2>/dev/null & +mount -t sysfs sysfs /sys 2>/dev/null & +mount -t tmpfs tmpfs /tmp 2>/dev/null & +mount -t tmpfs tmpfs /run 2>/dev/null & +mount -t devtmpfs devtmpfs /dev 2>/dev/null & +wait + +mkdir -p /dev/pts /dev/shm /sys/fs/cgroup /sandbox +mount -t devpts devpts /dev/pts 2>/dev/null & +mount -t tmpfs tmpfs /dev/shm 2>/dev/null & +mount -t cgroup2 cgroup2 /sys/fs/cgroup 2>/dev/null & +wait + +mount -t tmpfs tmpfs /sandbox 2>/dev/null || true +mkdir -p /sandbox +chown sandbox:sandbox /sandbox 2>/dev/null || true + +hostname openshell-sandbox-vm 2>/dev/null || true +ip link set lo up 2>/dev/null || true + +if ip link show eth0 >/dev/null 2>&1; then + ts "detected eth0 (gvproxy networking)" + ip link set eth0 up 2>/dev/null || true + + if command -v udhcpc >/dev/null 2>&1; then + UDHCPC_SCRIPT="/usr/share/udhcpc/default.script" + if [ ! -f "$UDHCPC_SCRIPT" ]; then + mkdir -p /usr/share/udhcpc + cat > "$UDHCPC_SCRIPT" <<'DHCP_SCRIPT' +#!/bin/sh +case "$1" in + bound|renew) + ip addr flush dev "$interface" + ip addr add "$ip/$mask" dev "$interface" + if [ -n "$router" ]; then + ip route add default via "$router" dev "$interface" + fi + if [ -n "$dns" ]; then + : > /etc/resolv.conf + for d in $dns; do + echo "nameserver $d" >> /etc/resolv.conf + done + fi + ;; +esac +DHCP_SCRIPT + chmod +x "$UDHCPC_SCRIPT" + fi + + if ! udhcpc -i eth0 -f -q -n -T 1 -t 3 -A 1 -s "$UDHCPC_SCRIPT" 2>&1; then + ts "WARNING: DHCP failed, falling back to static config" + ip addr add 192.168.127.2/24 dev eth0 2>/dev/null || true + ip route add default via 192.168.127.1 2>/dev/null || true + fi + else + ts "no DHCP client, using static config" + ip addr add 192.168.127.2/24 dev eth0 2>/dev/null || true + ip route add default via 192.168.127.1 2>/dev/null || true + fi + + if [ ! -s /etc/resolv.conf ]; then + echo "nameserver 8.8.8.8" > /etc/resolv.conf + echo "nameserver 8.8.4.4" >> /etc/resolv.conf + fi +else + ts "WARNING: eth0 not found; supervisor will start without guest egress" +fi + +export HOME=/sandbox +export USER=sandbox + +rewrite_openshell_endpoint_if_needed + +ts "starting openshell-sandbox supervisor" +exec /opt/openshell/bin/openshell-sandbox --workdir /sandbox diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs new file mode 100644 index 0000000000..3d3fbf4b61 --- /dev/null +++ b/crates/openshell-driver-vm/src/driver.rs @@ -0,0 +1,1363 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use crate::{ + GUEST_SSH_PORT, + rootfs::{extract_sandbox_rootfs_to, sandbox_guest_init_path}, +}; +use futures::Stream; +use nix::errno::Errno; +use nix::sys::signal::{Signal, kill}; +use nix::unistd::Pid; +use openshell_core::proto::compute::v1::{ + CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, + DriverCondition as SandboxCondition, DriverPlatformEvent as PlatformEvent, + DriverSandbox as Sandbox, DriverSandboxStatus as SandboxStatus, GetCapabilitiesRequest, + GetCapabilitiesResponse, GetSandboxRequest, GetSandboxResponse, ListSandboxesRequest, + ListSandboxesResponse, ResolveSandboxEndpointRequest, ResolveSandboxEndpointResponse, + SandboxEndpoint, StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, + ValidateSandboxCreateResponse, WatchSandboxesDeletedEvent, WatchSandboxesEvent, + WatchSandboxesPlatformEvent, WatchSandboxesRequest, WatchSandboxesSandboxEvent, + compute_driver_server::ComputeDriver, sandbox_endpoint, watch_sandboxes_event, +}; +use std::collections::{HashMap, HashSet}; +use std::net::{Ipv4Addr, SocketAddr, TcpListener}; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::pin::Pin; +use std::process::Stdio; +use std::sync::Arc; +use std::time::Duration; +use tokio::net::TcpStream; +use tokio::process::{Child, Command}; +use tokio::sync::{Mutex, broadcast, mpsc}; +use tokio_stream::wrappers::ReceiverStream; +use tonic::{Request, Response, Status}; +use url::{Host, Url}; + +const DRIVER_NAME: &str = "openshell-driver-vm"; +const WATCH_BUFFER: usize = 256; +const DEFAULT_VCPUS: u8 = 2; +const DEFAULT_MEM_MIB: u32 = 2048; +const GUEST_TLS_DIR: &str = "/opt/openshell/tls"; +const GUEST_TLS_CA_PATH: &str = "/opt/openshell/tls/ca.crt"; +const GUEST_TLS_CERT_PATH: &str = "/opt/openshell/tls/tls.crt"; +const GUEST_TLS_KEY_PATH: &str = "/opt/openshell/tls/tls.key"; + +#[derive(Debug, Clone)] +struct VmDriverTlsPaths { + ca: PathBuf, + cert: PathBuf, + key: PathBuf, +} + +#[derive(Debug, Clone)] +pub struct VmDriverConfig { + pub openshell_endpoint: String, + pub state_dir: PathBuf, + pub launcher_bin: Option, + pub ssh_handshake_secret: String, + pub ssh_handshake_skew_secs: u64, + pub log_level: String, + pub krun_log_level: u32, + pub vcpus: u8, + pub mem_mib: u32, + pub guest_tls_ca: Option, + pub guest_tls_cert: Option, + pub guest_tls_key: Option, +} + +impl Default for VmDriverConfig { + fn default() -> Self { + Self { + openshell_endpoint: String::new(), + state_dir: PathBuf::from("target/openshell-vm-driver"), + launcher_bin: None, + ssh_handshake_secret: String::new(), + ssh_handshake_skew_secs: 300, + log_level: "info".to_string(), + krun_log_level: 1, + vcpus: DEFAULT_VCPUS, + mem_mib: DEFAULT_MEM_MIB, + guest_tls_ca: None, + guest_tls_cert: None, + guest_tls_key: None, + } + } +} + +impl VmDriverConfig { + fn requires_tls_materials(&self) -> bool { + self.openshell_endpoint.starts_with("https://") + } + + fn tls_paths(&self) -> Result, String> { + let provided = [ + self.guest_tls_ca.as_ref(), + self.guest_tls_cert.as_ref(), + self.guest_tls_key.as_ref(), + ]; + if provided.iter().all(Option::is_none) { + return if self.requires_tls_materials() { + Err( + "https:// openshell endpoint requires OPENSHELL_VM_TLS_CA, OPENSHELL_VM_TLS_CERT, and OPENSHELL_VM_TLS_KEY so sandbox VMs can authenticate to the gateway" + .to_string(), + ) + } else { + Ok(None) + }; + } + + let Some(ca) = self.guest_tls_ca.clone() else { + return Err( + "OPENSHELL_VM_TLS_CA is required when TLS materials are configured".to_string(), + ); + }; + let Some(cert) = self.guest_tls_cert.clone() else { + return Err( + "OPENSHELL_VM_TLS_CERT is required when TLS materials are configured".to_string(), + ); + }; + let Some(key) = self.guest_tls_key.clone() else { + return Err( + "OPENSHELL_VM_TLS_KEY is required when TLS materials are configured".to_string(), + ); + }; + + for path in [&ca, &cert, &key] { + if !path.is_file() { + return Err(format!( + "TLS material '{}' does not exist or is not a file", + path.display() + )); + } + } + + Ok(Some(VmDriverTlsPaths { ca, cert, key })) + } +} + +fn validate_openshell_endpoint(endpoint: &str) -> Result<(), String> { + let url = Url::parse(endpoint) + .map_err(|err| format!("invalid openshell endpoint '{endpoint}': {err}"))?; + let Some(host) = url.host() else { + return Err(format!("openshell endpoint '{endpoint}' is missing a host")); + }; + + let invalid_from_vm = match host { + Host::Domain(_) => false, + Host::Ipv4(ip) => ip.is_unspecified(), + Host::Ipv6(ip) => ip.is_unspecified(), + }; + + if invalid_from_vm { + return Err(format!( + "openshell endpoint '{endpoint}' is not reachable from sandbox VMs; use a concrete host such as 127.0.0.1, host.containers.internal, or another routable address" + )); + } + + Ok(()) +} + +#[derive(Debug)] +struct VmProcess { + child: Child, + deleting: bool, +} + +#[derive(Debug)] +struct SandboxRecord { + snapshot: Sandbox, + ssh_port: u16, + state_dir: PathBuf, + process: Arc>, +} + +#[derive(Debug, Clone)] +pub struct VmDriver { + config: VmDriverConfig, + launcher_bin: PathBuf, + registry: Arc>>, + events: broadcast::Sender, +} + +impl VmDriver { + pub async fn new(config: VmDriverConfig) -> Result { + if config.openshell_endpoint.trim().is_empty() { + return Err("openshell endpoint is required".to_string()); + } + validate_openshell_endpoint(&config.openshell_endpoint)?; + let _ = config.tls_paths()?; + + let state_root = config.state_dir.join("sandboxes"); + tokio::fs::create_dir_all(&state_root) + .await + .map_err(|err| { + format!( + "failed to create state dir '{}': {err}", + state_root.display() + ) + })?; + + let launcher_bin = if let Some(path) = config.launcher_bin.clone() { + path + } else { + std::env::current_exe() + .map_err(|err| format!("failed to resolve vm driver executable: {err}"))? + }; + + let (events, _) = broadcast::channel(WATCH_BUFFER); + Ok(Self { + config, + launcher_bin, + registry: Arc::new(Mutex::new(HashMap::new())), + events, + }) + } + + #[must_use] + pub fn capabilities(&self) -> GetCapabilitiesResponse { + GetCapabilitiesResponse { + driver_name: DRIVER_NAME.to_string(), + driver_version: openshell_core::VERSION.to_string(), + default_image: String::new(), + supports_gpu: false, + } + } + + pub async fn validate_sandbox(&self, sandbox: &Sandbox) -> Result<(), Status> { + validate_vm_sandbox(sandbox) + } + + pub async fn create_sandbox(&self, sandbox: &Sandbox) -> Result { + validate_vm_sandbox(sandbox)?; + + if self.registry.lock().await.contains_key(&sandbox.id) { + return Err(Status::already_exists("sandbox already exists")); + } + + let ssh_port = allocate_local_port()?; + let state_dir = sandbox_state_dir(&self.config.state_dir, &sandbox.id); + let rootfs = state_dir.join("rootfs"); + + tokio::fs::create_dir_all(&state_dir) + .await + .map_err(|err| Status::internal(format!("create state dir failed: {err}")))?; + + let tls_paths = self + .config + .tls_paths() + .map_err(Status::failed_precondition)?; + let rootfs_for_extract = rootfs.clone(); + tokio::task::spawn_blocking(move || extract_sandbox_rootfs_to(&rootfs_for_extract)) + .await + .map_err(|err| Status::internal(format!("sandbox rootfs extraction panicked: {err}")))? + .map_err(|err| Status::internal(format!("extract sandbox rootfs failed: {err}")))?; + if let Some(tls_paths) = tls_paths.as_ref() { + prepare_guest_tls_materials(&rootfs, tls_paths) + .await + .map_err(|err| { + Status::internal(format!("prepare guest TLS materials failed: {err}")) + })?; + } + + let console_output = state_dir.join("rootfs-console.log"); + let mut command = Command::new(&self.launcher_bin); + command.kill_on_drop(true); + command.stdin(Stdio::null()); + command.stdout(Stdio::inherit()); + command.stderr(Stdio::inherit()); + command.arg("--internal-run-vm"); + command.arg("--vm-rootfs").arg(&rootfs); + command.arg("--vm-exec").arg(sandbox_guest_init_path()); + command.arg("--vm-workdir").arg("/"); + command.arg("--vm-vcpus").arg(self.config.vcpus.to_string()); + command + .arg("--vm-mem-mib") + .arg(self.config.mem_mib.to_string()); + command + .arg("--vm-krun-log-level") + .arg(self.config.krun_log_level.to_string()); + command.arg("--vm-console-output").arg(&console_output); + command + .arg("--vm-port") + .arg(format!("{ssh_port}:{GUEST_SSH_PORT}")); + for env in build_guest_environment(sandbox, &self.config) { + command.arg("--vm-env").arg(env); + } + + let child = match command.spawn() { + Ok(child) => child, + Err(err) => { + let _ = tokio::fs::remove_dir_all(&state_dir).await; + return Err(Status::internal(format!( + "failed to launch vm helper '{}': {err}", + self.launcher_bin.display() + ))); + } + }; + let snapshot = sandbox_snapshot(sandbox, provisioning_condition(), false); + let process = Arc::new(Mutex::new(VmProcess { + child, + deleting: false, + })); + + { + let mut registry = self.registry.lock().await; + registry.insert( + sandbox.id.clone(), + SandboxRecord { + snapshot: snapshot.clone(), + ssh_port, + state_dir: state_dir.clone(), + process: process.clone(), + }, + ); + } + + self.publish_snapshot(snapshot.clone()); + tokio::spawn({ + let driver = self.clone(); + let sandbox_id = sandbox.id.clone(); + async move { + driver.monitor_sandbox(sandbox_id).await; + } + }); + + Ok(CreateSandboxResponse {}) + } + + pub async fn delete_sandbox( + &self, + sandbox_id: &str, + sandbox_name: &str, + ) -> Result { + let record = { + let registry = self.registry.lock().await; + if let Some((id, record)) = registry.get_key_value(sandbox_id) { + Some((id.clone(), record.state_dir.clone(), record.process.clone())) + } else { + let matched_id = registry + .iter() + .find(|(_, record)| record.snapshot.name == sandbox_name) + .map(|(id, _)| id.clone()); + matched_id.and_then(|id| { + registry + .get(&id) + .map(|record| (id, record.state_dir.clone(), record.process.clone())) + }) + } + }; + + let Some((record_id, state_dir, process)) = record else { + return Ok(DeleteSandboxResponse { deleted: false }); + }; + + if let Some(snapshot) = self + .set_snapshot_condition(&record_id, deleting_condition(), true) + .await + { + self.publish_snapshot(snapshot); + } + + { + let mut process = process.lock().await; + process.deleting = true; + terminate_vm_process(&mut process.child) + .await + .map_err(|err| Status::internal(format!("failed to stop vm: {err}")))?; + } + + if let Err(err) = tokio::fs::remove_dir_all(&state_dir).await + && err.kind() != std::io::ErrorKind::NotFound + { + return Err(Status::internal(format!( + "failed to remove state dir: {err}" + ))); + } + + { + let mut registry = self.registry.lock().await; + registry.remove(&record_id); + } + + self.publish_deleted(record_id); + Ok(DeleteSandboxResponse { deleted: true }) + } + + pub async fn resolve_endpoint( + &self, + sandbox: &Sandbox, + ) -> Result { + let registry = self.registry.lock().await; + let record = registry.get(&sandbox.id).or_else(|| { + registry + .values() + .find(|record| record.snapshot.name == sandbox.name) + }); + let record = record.ok_or_else(|| Status::not_found("sandbox not found"))?; + Ok(ResolveSandboxEndpointResponse { + endpoint: Some(SandboxEndpoint { + target: Some(sandbox_endpoint::Target::Host("127.0.0.1".to_string())), + port: u32::from(record.ssh_port), + }), + }) + } + + pub async fn get_sandbox( + &self, + sandbox_id: &str, + sandbox_name: &str, + ) -> Result, Status> { + let registry = self.registry.lock().await; + let sandbox = if !sandbox_id.is_empty() { + registry + .get(sandbox_id) + .map(|record| record.snapshot.clone()) + } else { + registry + .values() + .find(|record| record.snapshot.name == sandbox_name) + .map(|record| record.snapshot.clone()) + }; + Ok(sandbox) + } + + pub async fn current_snapshots(&self) -> Vec { + let registry = self.registry.lock().await; + let mut snapshots = registry + .values() + .map(|record| record.snapshot.clone()) + .collect::>(); + snapshots.sort_by(|left, right| left.name.cmp(&right.name)); + snapshots + } + + async fn monitor_sandbox(&self, sandbox_id: String) { + let mut ready_emitted = false; + + loop { + let (process, ssh_port, state_dir) = { + let registry = self.registry.lock().await; + let Some(record) = registry.get(&sandbox_id) else { + return; + }; + ( + record.process.clone(), + record.ssh_port, + record.state_dir.clone(), + ) + }; + + let exit_status = { + let mut process = process.lock().await; + if process.deleting { + return; + } + match process.child.try_wait() { + Ok(status) => status, + Err(err) => { + if let Some(snapshot) = self + .set_snapshot_condition( + &sandbox_id, + error_condition("ProcessPollFailed", &err.to_string()), + false, + ) + .await + { + self.publish_snapshot(snapshot); + } + self.publish_platform_event( + sandbox_id.clone(), + platform_event( + "vm", + "Warning", + "ProcessPollFailed", + format!("Failed to poll VM helper process: {err}"), + ), + ); + return; + } + } + }; + + if let Some(status) = exit_status { + let message = match status.code() { + Some(code) => format!("VM process exited with status {code}"), + None => "VM process exited".to_string(), + }; + if let Some(snapshot) = self + .set_snapshot_condition( + &sandbox_id, + error_condition("ProcessExited", &message), + false, + ) + .await + { + self.publish_snapshot(snapshot); + } + self.publish_platform_event( + sandbox_id.clone(), + platform_event("vm", "Warning", "ProcessExited", message), + ); + return; + } + + if !ready_emitted && port_is_ready(ssh_port).await && guest_ssh_ready(&state_dir).await + { + if let Some(snapshot) = self + .set_snapshot_condition(&sandbox_id, ready_condition(), false) + .await + { + self.publish_snapshot(snapshot); + } + ready_emitted = true; + } + + tokio::time::sleep(Duration::from_millis(250)).await; + } + } + + async fn set_snapshot_condition( + &self, + sandbox_id: &str, + condition: SandboxCondition, + deleting: bool, + ) -> Option { + let mut registry = self.registry.lock().await; + let record = registry.get_mut(sandbox_id)?; + record.snapshot.status = Some(status_with_condition(&record.snapshot, condition, deleting)); + Some(record.snapshot.clone()) + } + + fn publish_snapshot(&self, sandbox: Sandbox) { + let _ = self.events.send(WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Sandbox( + WatchSandboxesSandboxEvent { + sandbox: Some(sandbox), + }, + )), + }); + } + + fn publish_deleted(&self, sandbox_id: String) { + let _ = self.events.send(WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Deleted( + WatchSandboxesDeletedEvent { sandbox_id }, + )), + }); + } + + fn publish_platform_event(&self, sandbox_id: String, event: PlatformEvent) { + let _ = self.events.send(WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::PlatformEvent( + WatchSandboxesPlatformEvent { + sandbox_id, + event: Some(event), + }, + )), + }); + } +} + +#[tonic::async_trait] +impl ComputeDriver for VmDriver { + async fn get_capabilities( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(self.capabilities())) + } + + async fn validate_sandbox_create( + &self, + request: Request, + ) -> Result, Status> { + let sandbox = request + .into_inner() + .sandbox + .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; + self.validate_sandbox(&sandbox).await?; + Ok(Response::new(ValidateSandboxCreateResponse {})) + } + + async fn create_sandbox( + &self, + request: Request, + ) -> Result, Status> { + let sandbox = request + .into_inner() + .sandbox + .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; + let response = self.create_sandbox(&sandbox).await?; + Ok(Response::new(response)) + } + + async fn get_sandbox( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + if request.sandbox_id.is_empty() && request.sandbox_name.is_empty() { + return Err(Status::invalid_argument( + "sandbox_id or sandbox_name is required", + )); + } + + let sandbox = self + .get_sandbox(&request.sandbox_id, &request.sandbox_name) + .await? + .ok_or_else(|| Status::not_found("sandbox not found"))?; + + if !request.sandbox_id.is_empty() && request.sandbox_id != sandbox.id { + return Err(Status::failed_precondition( + "sandbox_id did not match the fetched sandbox", + )); + } + + Ok(Response::new(GetSandboxResponse { + sandbox: Some(sandbox), + })) + } + + async fn list_sandboxes( + &self, + _request: Request, + ) -> Result, Status> { + Ok(Response::new(ListSandboxesResponse { + sandboxes: self.current_snapshots().await, + })) + } + + async fn stop_sandbox( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented( + "stop sandbox is not implemented by the vm compute driver", + )) + } + + async fn delete_sandbox( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + let response = self + .delete_sandbox(&request.sandbox_id, &request.sandbox_name) + .await?; + Ok(Response::new(response)) + } + + async fn resolve_sandbox_endpoint( + &self, + request: Request, + ) -> Result, Status> { + let sandbox = request + .into_inner() + .sandbox + .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; + Ok(Response::new(self.resolve_endpoint(&sandbox).await?)) + } + + type WatchSandboxesStream = + Pin> + Send + 'static>>; + + async fn watch_sandboxes( + &self, + _request: Request, + ) -> Result, Status> { + let initial = self.current_snapshots().await; + let mut rx = self.events.subscribe(); + let (tx, out_rx) = mpsc::channel(WATCH_BUFFER); + tokio::spawn(async move { + let mut sent = HashSet::new(); + for sandbox in initial { + sent.insert(sandbox.id.clone()); + if tx + .send(Ok(WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Sandbox( + WatchSandboxesSandboxEvent { + sandbox: Some(sandbox), + }, + )), + })) + .await + .is_err() + { + return; + } + } + + loop { + match rx.recv().await { + Ok(event) => { + if let Some(watch_sandboxes_event::Payload::Sandbox(sandbox_event)) = + &event.payload + && let Some(sandbox) = &sandbox_event.sandbox + && !sent.insert(sandbox.id.clone()) + { + // duplicate snapshots are still forwarded + } + if tx.send(Ok(event)).await.is_err() { + return; + } + } + Err(broadcast::error::RecvError::Lagged(_)) => continue, + Err(broadcast::error::RecvError::Closed) => return, + } + } + }); + + Ok(Response::new(Box::pin(ReceiverStream::new(out_rx)))) + } +} + +fn validate_vm_sandbox(sandbox: &Sandbox) -> Result<(), Status> { + let spec = sandbox + .spec + .as_ref() + .ok_or_else(|| Status::invalid_argument("sandbox spec is required"))?; + if spec.gpu { + return Err(Status::failed_precondition( + "vm sandboxes do not support gpu=true", + )); + } + if let Some(template) = spec.template.as_ref() { + if !template.image.is_empty() { + return Err(Status::failed_precondition( + "vm sandboxes do not support template.image", + )); + } + if !template.agent_socket_path.is_empty() { + return Err(Status::failed_precondition( + "vm sandboxes do not support template.agent_socket_path", + )); + } + if template.platform_config.is_some() { + return Err(Status::failed_precondition( + "vm sandboxes do not support template.platform_config", + )); + } + if template.resources.is_some() { + return Err(Status::failed_precondition( + "vm sandboxes do not support template.resources", + )); + } + } + Ok(()) +} + +fn merged_environment(sandbox: &Sandbox) -> HashMap { + let mut environment = sandbox + .spec + .as_ref() + .and_then(|spec| spec.template.as_ref()) + .map_or_else(HashMap::new, |template| template.environment.clone()); + if let Some(spec) = sandbox.spec.as_ref() { + environment.extend(spec.environment.clone()); + } + environment +} + +fn guest_visible_openshell_endpoint(endpoint: &str) -> String { + let Ok(mut url) = Url::parse(endpoint) else { + return endpoint.to_string(); + }; + + let should_rewrite = match url.host() { + Some(Host::Ipv4(ip)) => ip.is_loopback(), + Some(Host::Ipv6(ip)) => ip.is_loopback(), + Some(Host::Domain(host)) => host.eq_ignore_ascii_case("localhost"), + None => false, + }; + + if should_rewrite && url.set_host(Some("192.168.127.1")).is_ok() { + return url.to_string(); + } + + endpoint.to_string() +} + +fn build_guest_environment(sandbox: &Sandbox, config: &VmDriverConfig) -> Vec { + let mut environment = HashMap::from([ + ("HOME".to_string(), "/root".to_string()), + ( + "PATH".to_string(), + "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin".to_string(), + ), + ("TERM".to_string(), "xterm".to_string()), + ( + "OPENSHELL_ENDPOINT".to_string(), + guest_visible_openshell_endpoint(&config.openshell_endpoint), + ), + ("OPENSHELL_SANDBOX_ID".to_string(), sandbox.id.clone()), + ("OPENSHELL_SANDBOX".to_string(), sandbox.name.clone()), + ( + "OPENSHELL_SSH_LISTEN_ADDR".to_string(), + format!("0.0.0.0:{GUEST_SSH_PORT}"), + ), + ( + "OPENSHELL_SSH_HANDSHAKE_SECRET".to_string(), + config.ssh_handshake_secret.clone(), + ), + ( + "OPENSHELL_SSH_HANDSHAKE_SKEW_SECS".to_string(), + config.ssh_handshake_skew_secs.to_string(), + ), + ( + "OPENSHELL_SANDBOX_COMMAND".to_string(), + "tail -f /dev/null".to_string(), + ), + ( + "OPENSHELL_LOG_LEVEL".to_string(), + sandbox_log_level(sandbox, &config.log_level), + ), + ]); + if config.requires_tls_materials() { + environment.extend(HashMap::from([ + ( + "OPENSHELL_TLS_CA".to_string(), + GUEST_TLS_CA_PATH.to_string(), + ), + ( + "OPENSHELL_TLS_CERT".to_string(), + GUEST_TLS_CERT_PATH.to_string(), + ), + ( + "OPENSHELL_TLS_KEY".to_string(), + GUEST_TLS_KEY_PATH.to_string(), + ), + ])); + } + environment.extend(merged_environment(sandbox)); + + let mut pairs = environment.into_iter().collect::>(); + pairs.sort_by(|left, right| left.0.cmp(&right.0)); + pairs + .into_iter() + .map(|(key, value)| format!("{key}={value}")) + .collect() +} + +fn sandbox_log_level(sandbox: &Sandbox, default_level: &str) -> String { + sandbox + .spec + .as_ref() + .map(|spec| spec.log_level.as_str()) + .filter(|level| !level.is_empty()) + .unwrap_or(default_level) + .to_string() +} + +fn sandbox_state_dir(root: &Path, sandbox_id: &str) -> PathBuf { + root.join("sandboxes").join(sandbox_id) +} + +async fn prepare_guest_tls_materials( + rootfs: &Path, + paths: &VmDriverTlsPaths, +) -> Result<(), std::io::Error> { + let guest_tls_dir = rootfs.join(GUEST_TLS_DIR.trim_start_matches('/')); + tokio::fs::create_dir_all(&guest_tls_dir).await?; + + copy_guest_tls_material(&paths.ca, &guest_tls_dir.join("ca.crt"), 0o644).await?; + copy_guest_tls_material(&paths.cert, &guest_tls_dir.join("tls.crt"), 0o644).await?; + copy_guest_tls_material(&paths.key, &guest_tls_dir.join("tls.key"), 0o600).await?; + Ok(()) +} + +async fn copy_guest_tls_material( + source: &Path, + dest: &Path, + mode: u32, +) -> Result<(), std::io::Error> { + tokio::fs::copy(source, dest).await?; + tokio::fs::set_permissions(dest, std::fs::Permissions::from_mode(mode)).await?; + Ok(()) +} + +async fn terminate_vm_process(child: &mut Child) -> Result<(), std::io::Error> { + if let Some(pid) = child.id() + && let Err(err) = kill(Pid::from_raw(pid as i32), Signal::SIGTERM) + && err != Errno::ESRCH + { + return Err(std::io::Error::other(format!( + "send SIGTERM to vm process {pid}: {err}" + ))); + } + + match tokio::time::timeout(Duration::from_secs(5), child.wait()).await { + Ok(Ok(_)) => Ok(()), + Ok(Err(err)) => Err(err), + Err(_) => { + child.kill().await?; + child.wait().await.map(|_| ()) + } + } +} + +fn allocate_local_port() -> Result { + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)) + .map_err(|err| Status::internal(format!("failed to allocate local ssh port: {err}")))?; + listener + .local_addr() + .map(|addr| addr.port()) + .map_err(|err| Status::internal(format!("failed to inspect local ssh port: {err}"))) +} + +async fn port_is_ready(port: u16) -> bool { + TcpStream::connect(SocketAddr::new(Ipv4Addr::LOCALHOST.into(), port)) + .await + .is_ok() +} + +async fn guest_ssh_ready(state_dir: &Path) -> bool { + let console_log = state_dir.join("rootfs-console.log"); + let Ok(contents) = tokio::fs::read_to_string(console_log).await else { + return false; + }; + + contents.contains("SSH server is ready to accept connections") + || contents.contains("SSH server listening") +} + +fn sandbox_snapshot(sandbox: &Sandbox, condition: SandboxCondition, deleting: bool) -> Sandbox { + Sandbox { + id: sandbox.id.clone(), + name: sandbox.name.clone(), + namespace: sandbox.namespace.clone(), + status: Some(SandboxStatus { + sandbox_name: sandbox.name.clone(), + instance_id: String::new(), + agent_fd: String::new(), + sandbox_fd: String::new(), + conditions: vec![condition], + deleting, + }), + ..Default::default() + } +} + +fn status_with_condition( + snapshot: &Sandbox, + condition: SandboxCondition, + deleting: bool, +) -> SandboxStatus { + SandboxStatus { + sandbox_name: snapshot.name.clone(), + instance_id: String::new(), + agent_fd: String::new(), + sandbox_fd: String::new(), + conditions: vec![condition], + deleting, + } +} + +fn provisioning_condition() -> SandboxCondition { + SandboxCondition { + r#type: "Ready".to_string(), + status: "False".to_string(), + reason: "Starting".to_string(), + message: "VM is starting".to_string(), + last_transition_time: String::new(), + } +} + +fn ready_condition() -> SandboxCondition { + SandboxCondition { + r#type: "Ready".to_string(), + status: "True".to_string(), + reason: "Listening".to_string(), + message: "Supervisor is listening for SSH connections".to_string(), + last_transition_time: String::new(), + } +} + +fn deleting_condition() -> SandboxCondition { + SandboxCondition { + r#type: "Ready".to_string(), + status: "False".to_string(), + reason: "Deleting".to_string(), + message: "Sandbox is being deleted".to_string(), + last_transition_time: String::new(), + } +} + +fn error_condition(reason: &str, message: &str) -> SandboxCondition { + SandboxCondition { + r#type: "Ready".to_string(), + status: "False".to_string(), + reason: reason.to_string(), + message: message.to_string(), + last_transition_time: String::new(), + } +} + +fn platform_event(source: &str, event_type: &str, reason: &str, message: String) -> PlatformEvent { + PlatformEvent { + timestamp_ms: current_time_ms(), + source: source.to_string(), + r#type: event_type.to_string(), + reason: reason.to_string(), + message, + metadata: HashMap::new(), + } +} + +fn current_time_ms() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |duration| duration.as_millis() as i64) +} + +#[cfg(test)] +mod tests { + use super::*; + use openshell_core::proto::compute::v1::{ + DriverSandboxSpec as SandboxSpec, DriverSandboxTemplate as SandboxTemplate, + }; + use prost_types::{Struct, Value, value::Kind}; + use std::sync::atomic::{AtomicU64, Ordering}; + use std::time::{SystemTime, UNIX_EPOCH}; + use tonic::Code; + + #[test] + fn validate_vm_sandbox_rejects_gpu() { + let sandbox = Sandbox { + spec: Some(SandboxSpec { + gpu: true, + ..Default::default() + }), + ..Default::default() + }; + let err = validate_vm_sandbox(&sandbox).expect_err("gpu should be rejected"); + assert_eq!(err.code(), Code::FailedPrecondition); + assert!(err.message().contains("gpu")); + } + + #[test] + fn validate_vm_sandbox_rejects_platform_config() { + let sandbox = Sandbox { + spec: Some(SandboxSpec { + template: Some(SandboxTemplate { + platform_config: Some(Struct { + fields: [( + "runtime_class_name".to_string(), + Value { + kind: Some(Kind::StringValue("kata".to_string())), + }, + )] + .into_iter() + .collect(), + }), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + let err = validate_vm_sandbox(&sandbox).expect_err("platform config should be rejected"); + assert_eq!(err.code(), Code::FailedPrecondition); + assert!(err.message().contains("platform_config")); + } + + #[test] + fn merged_environment_prefers_spec_values() { + let sandbox = Sandbox { + spec: Some(SandboxSpec { + environment: HashMap::from([("A".to_string(), "spec".to_string())]), + template: Some(SandboxTemplate { + environment: HashMap::from([ + ("A".to_string(), "template".to_string()), + ("B".to_string(), "template".to_string()), + ]), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }; + let merged = merged_environment(&sandbox); + assert_eq!(merged.get("A"), Some(&"spec".to_string())); + assert_eq!(merged.get("B"), Some(&"template".to_string())); + } + + #[test] + fn build_guest_environment_sets_supervisor_defaults() { + let config = VmDriverConfig { + openshell_endpoint: "http://127.0.0.1:8080".to_string(), + ssh_handshake_secret: "secret".to_string(), + ..Default::default() + }; + let sandbox = Sandbox { + id: "sandbox-123".to_string(), + name: "sandbox-123".to_string(), + spec: Some(SandboxSpec::default()), + ..Default::default() + }; + + let env = build_guest_environment(&sandbox, &config); + assert!(env.contains(&"HOME=/root".to_string())); + assert!(env.contains(&"OPENSHELL_ENDPOINT=http://192.168.127.1:8080/".to_string())); + assert!(env.contains(&"OPENSHELL_SANDBOX_ID=sandbox-123".to_string())); + assert!(env.contains(&format!( + "OPENSHELL_SSH_LISTEN_ADDR=0.0.0.0:{GUEST_SSH_PORT}" + ))); + } + + #[test] + fn guest_visible_openshell_endpoint_preserves_non_loopback_hosts() { + assert_eq!( + guest_visible_openshell_endpoint("http://host.containers.internal:8080"), + "http://host.containers.internal:8080" + ); + assert_eq!( + guest_visible_openshell_endpoint("https://gateway.internal:8443"), + "https://gateway.internal:8443" + ); + } + + #[test] + fn build_guest_environment_includes_tls_paths_for_https_endpoint() { + let config = VmDriverConfig { + openshell_endpoint: "https://127.0.0.1:8443".to_string(), + ssh_handshake_secret: "secret".to_string(), + guest_tls_ca: Some(PathBuf::from("/host/ca.crt")), + guest_tls_cert: Some(PathBuf::from("/host/tls.crt")), + guest_tls_key: Some(PathBuf::from("/host/tls.key")), + ..Default::default() + }; + let sandbox = Sandbox { + id: "sandbox-123".to_string(), + name: "sandbox-123".to_string(), + spec: Some(SandboxSpec::default()), + ..Default::default() + }; + + let env = build_guest_environment(&sandbox, &config); + assert!(env.contains(&format!("OPENSHELL_TLS_CA={GUEST_TLS_CA_PATH}"))); + assert!(env.contains(&format!("OPENSHELL_TLS_CERT={GUEST_TLS_CERT_PATH}"))); + assert!(env.contains(&format!("OPENSHELL_TLS_KEY={GUEST_TLS_KEY_PATH}"))); + } + + #[test] + fn vm_driver_config_requires_tls_materials_for_https_endpoint() { + let config = VmDriverConfig { + openshell_endpoint: "https://127.0.0.1:8443".to_string(), + ..Default::default() + }; + let err = config + .tls_paths() + .expect_err("https endpoint should require TLS materials"); + assert!(err.contains("OPENSHELL_VM_TLS_CA")); + } + + #[tokio::test] + async fn delete_sandbox_keeps_registry_entry_when_cleanup_fails() { + let (events, _) = broadcast::channel(WATCH_BUFFER); + let driver = VmDriver { + config: VmDriverConfig::default(), + launcher_bin: PathBuf::from("openshell-driver-vm"), + registry: Arc::new(Mutex::new(HashMap::new())), + events, + }; + + let base = unique_temp_dir(); + std::fs::create_dir_all(&base).unwrap(); + let state_file = base.join("state-file"); + std::fs::write(&state_file, "not a directory").unwrap(); + + insert_test_record( + &driver, + "sandbox-123", + state_file.clone(), + spawn_exited_child(), + ) + .await; + + let err = driver + .delete_sandbox("sandbox-123", "sandbox-123") + .await + .expect_err("state dir cleanup should fail for a file path"); + assert!(err.message().contains("failed to remove state dir")); + assert!(driver.registry.lock().await.contains_key("sandbox-123")); + + let retry_state_dir = base.join("state-dir"); + std::fs::create_dir_all(&retry_state_dir).unwrap(); + { + let mut registry = driver.registry.lock().await; + let record = registry.get_mut("sandbox-123").unwrap(); + record.state_dir = retry_state_dir; + record.process = Arc::new(Mutex::new(VmProcess { + child: spawn_exited_child(), + deleting: false, + })); + } + + let response = driver + .delete_sandbox("sandbox-123", "sandbox-123") + .await + .expect("delete retry should succeed once cleanup works"); + assert!(response.deleted); + assert!(!driver.registry.lock().await.contains_key("sandbox-123")); + + let _ = std::fs::remove_dir_all(base); + } + + #[test] + fn validate_openshell_endpoint_accepts_loopback_hosts() { + validate_openshell_endpoint("http://127.0.0.1:8080") + .expect("ipv4 loopback should be allowed for TSI"); + validate_openshell_endpoint("http://localhost:8080") + .expect("localhost should be allowed for TSI"); + validate_openshell_endpoint("http://[::1]:8080") + .expect("ipv6 loopback should be allowed for TSI"); + } + + #[test] + fn validate_openshell_endpoint_rejects_unspecified_hosts() { + let err = validate_openshell_endpoint("http://0.0.0.0:8080") + .expect_err("unspecified endpoint should fail"); + assert!(err.contains("not reachable from sandbox VMs")); + } + + #[test] + fn validate_openshell_endpoint_accepts_host_gateway() { + validate_openshell_endpoint("http://host.containers.internal:8080") + .expect("guest-reachable host alias should be accepted"); + validate_openshell_endpoint("http://192.168.127.1:8080") + .expect("gateway IP should be accepted"); + validate_openshell_endpoint("http://host.openshell.internal:8080") + .expect("openshell host alias should be accepted"); + validate_openshell_endpoint("https://gateway.internal:8443") + .expect("dns endpoint should be accepted"); + } + + #[tokio::test] + async fn prepare_guest_tls_materials_copies_bundle_into_rootfs() { + let base = unique_temp_dir(); + let source_dir = base.join("source"); + let rootfs = base.join("rootfs"); + std::fs::create_dir_all(&source_dir).unwrap(); + std::fs::create_dir_all(&rootfs).unwrap(); + + let ca = source_dir.join("ca.crt"); + let cert = source_dir.join("tls.crt"); + let key = source_dir.join("tls.key"); + std::fs::write(&ca, "ca").unwrap(); + std::fs::write(&cert, "cert").unwrap(); + std::fs::write(&key, "key").unwrap(); + + prepare_guest_tls_materials( + &rootfs, + &VmDriverTlsPaths { + ca: ca.clone(), + cert: cert.clone(), + key: key.clone(), + }, + ) + .await + .unwrap(); + + let guest_dir = rootfs.join(GUEST_TLS_DIR.trim_start_matches('/')); + assert_eq!( + std::fs::read_to_string(guest_dir.join("ca.crt")).unwrap(), + "ca" + ); + assert_eq!( + std::fs::read_to_string(guest_dir.join("tls.crt")).unwrap(), + "cert" + ); + assert_eq!( + std::fs::read_to_string(guest_dir.join("tls.key")).unwrap(), + "key" + ); + let key_mode = std::fs::metadata(guest_dir.join("tls.key")) + .unwrap() + .permissions() + .mode() + & 0o777; + assert_eq!(key_mode, 0o600); + + let _ = std::fs::remove_dir_all(base); + } + + #[tokio::test] + async fn guest_ssh_ready_detects_guest_console_marker() { + let base = unique_temp_dir(); + std::fs::create_dir_all(&base).unwrap(); + std::fs::write( + base.join("rootfs-console.log"), + "...\nINFO openshell_sandbox: SSH server is ready to accept connections\n", + ) + .unwrap(); + + assert!(guest_ssh_ready(&base).await); + + let _ = std::fs::remove_dir_all(base); + } + + #[tokio::test] + async fn guest_ssh_ready_is_false_without_marker() { + let base = unique_temp_dir(); + std::fs::create_dir_all(&base).unwrap(); + std::fs::write(base.join("rootfs-console.log"), "sandbox booting\n").unwrap(); + + assert!(!guest_ssh_ready(&base).await); + + let _ = std::fs::remove_dir_all(base); + } + + fn unique_temp_dir() -> PathBuf { + static COUNTER: AtomicU64 = AtomicU64::new(0); + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let suffix = COUNTER.fetch_add(1, Ordering::Relaxed); + std::env::temp_dir().join(format!( + "openshell-vm-driver-test-{}-{nanos}-{suffix}", + std::process::id() + )) + } + + fn spawn_exited_child() -> Child { + Command::new("sh") + .arg("-c") + .arg("exit 0") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap() + } + + async fn insert_test_record( + driver: &VmDriver, + sandbox_id: &str, + state_dir: PathBuf, + child: Child, + ) { + let sandbox = Sandbox { + id: sandbox_id.to_string(), + name: sandbox_id.to_string(), + ..Default::default() + }; + let process = Arc::new(Mutex::new(VmProcess { + child, + deleting: false, + })); + + let mut registry = driver.registry.lock().await; + registry.insert( + sandbox_id.to_string(), + SandboxRecord { + snapshot: sandbox, + ssh_port: 2222, + state_dir, + process, + }, + ); + } +} diff --git a/crates/openshell-driver-vm/src/embedded_runtime.rs b/crates/openshell-driver-vm/src/embedded_runtime.rs new file mode 100644 index 0000000000..63f83b874a --- /dev/null +++ b/crates/openshell-driver-vm/src/embedded_runtime.rs @@ -0,0 +1,181 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Embedded libkrun runtime resources for the VM driver. + +use std::fs; +use std::path::{Path, PathBuf}; + +#[cfg(all(target_os = "macos", target_arch = "aarch64"))] +mod resources { + pub const LIBKRUN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/libkrun.dylib.zst")); + pub const LIBKRUNFW: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/libkrunfw.5.dylib.zst")); + pub const GVPROXY: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/gvproxy.zst")); + pub const LIBKRUN_NAME: &str = "libkrun.dylib"; + pub const LIBKRUNFW_NAME: &str = "libkrunfw.5.dylib"; +} + +#[cfg(all(target_os = "linux", target_arch = "aarch64"))] +mod resources { + pub const LIBKRUN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/libkrun.so.zst")); + pub const LIBKRUNFW: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/libkrunfw.so.5.zst")); + pub const GVPROXY: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/gvproxy.zst")); + pub const LIBKRUN_NAME: &str = "libkrun.so"; + pub const LIBKRUNFW_NAME: &str = "libkrunfw.so.5"; +} + +#[cfg(all(target_os = "linux", target_arch = "x86_64"))] +mod resources { + pub const LIBKRUN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/libkrun.so.zst")); + pub const LIBKRUNFW: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/libkrunfw.so.5.zst")); + pub const GVPROXY: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/gvproxy.zst")); + pub const LIBKRUN_NAME: &str = "libkrun.so"; + pub const LIBKRUNFW_NAME: &str = "libkrunfw.so.5"; +} + +#[cfg(not(any( + all(target_os = "macos", target_arch = "aarch64"), + all(target_os = "linux", target_arch = "aarch64"), + all(target_os = "linux", target_arch = "x86_64"), +)))] +mod resources { + pub const LIBKRUN: &[u8] = &[]; + pub const LIBKRUNFW: &[u8] = &[]; + pub const GVPROXY: &[u8] = &[]; + pub const LIBKRUN_NAME: &str = "libkrun"; + pub const LIBKRUNFW_NAME: &str = "libkrunfw"; +} + +const VERSION: &str = env!("CARGO_PKG_VERSION"); + +pub fn ensure_runtime_extracted() -> Result { + if resources::LIBKRUN.is_empty() { + return Err( + "VM runtime not embedded for this platform. Supported: macOS ARM64, Linux ARM64, Linux x86_64" + .to_string(), + ); + } + + let cache_dir = runtime_cache_dir()?; + let version_marker = cache_dir.join(".version"); + let cache_key = runtime_cache_key(); + + if version_marker.exists() + && let Ok(cached_key) = fs::read_to_string(&version_marker) + && cached_key.trim() == cache_key + && validate_runtime_dir(&cache_dir).is_ok() + { + return Ok(cache_dir); + } + + cleanup_old_versions(&cache_dir)?; + + if cache_dir.exists() { + fs::remove_dir_all(&cache_dir) + .map_err(|e| format!("remove old runtime cache {}: {e}", cache_dir.display()))?; + } + fs::create_dir_all(&cache_dir) + .map_err(|e| format!("create runtime cache {}: {e}", cache_dir.display()))?; + + extract_resource(resources::LIBKRUN, &cache_dir.join(resources::LIBKRUN_NAME))?; + extract_resource( + resources::LIBKRUNFW, + &cache_dir.join(resources::LIBKRUNFW_NAME), + )?; + extract_resource(resources::GVPROXY, &cache_dir.join("gvproxy"))?; + + #[cfg(target_os = "macos")] + { + let unversioned = cache_dir.join("libkrunfw.dylib"); + if !unversioned.exists() { + std::os::unix::fs::symlink(resources::LIBKRUNFW_NAME, &unversioned) + .map_err(|e| format!("symlink {}: {e}", unversioned.display()))?; + } + } + + fs::write(&version_marker, cache_key) + .map_err(|e| format!("write runtime marker {}: {e}", version_marker.display()))?; + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + fs::set_permissions(cache_dir.join("gvproxy"), fs::Permissions::from_mode(0o755)) + .map_err(|e| format!("chmod gvproxy: {e}"))?; + } + + Ok(cache_dir) +} + +pub fn validate_runtime_dir(dir: &Path) -> Result<(), String> { + let libkrun = dir.join(resources::LIBKRUN_NAME); + let libkrunfw = dir.join(resources::LIBKRUNFW_NAME); + let gvproxy = dir.join("gvproxy"); + + for path in [&libkrun, &libkrunfw, &gvproxy] { + if !path.is_file() { + return Err(format!("missing runtime file: {}", path.display())); + } + let size = fs::metadata(path).map(|m| m.len()).unwrap_or(0); + if size == 0 { + return Err(format!("runtime file is empty (stub): {}", path.display())); + } + } + + Ok(()) +} + +fn runtime_cache_key() -> String { + let mut fp: u64 = 0; + for (index, chunk) in [resources::LIBKRUN, resources::LIBKRUNFW] + .into_iter() + .chain(std::iter::once(resources::GVPROXY)) + .enumerate() + { + let sample = &chunk[..chunk.len().min(64)]; + let mut word: u64 = 0; + for (offset, byte) in sample.iter().enumerate() { + word ^= (*byte as u64) << ((offset % 8) * 8); + } + fp ^= word.rotate_left((index as u32) * 13 + 7); + fp ^= (chunk.len() as u64).rotate_left((index as u32) * 17 + 3); + } + format!("{VERSION}-{fp:016x}") +} + +fn runtime_cache_dir() -> Result { + let base = + openshell_core::paths::xdg_data_dir().map_err(|e| format!("resolve XDG data dir: {e}"))?; + Ok(base.join("openshell").join("vm-runtime").join(VERSION)) +} + +fn runtime_cache_base() -> Result { + let base = + openshell_core::paths::xdg_data_dir().map_err(|e| format!("resolve XDG data dir: {e}"))?; + Ok(base.join("openshell").join("vm-runtime")) +} + +fn cleanup_old_versions(current_dir: &Path) -> Result<(), String> { + let base = runtime_cache_base()?; + if !base.exists() { + return Ok(()); + } + + let entries = fs::read_dir(&base).map_err(|e| format!("read {}: {e}", base.display()))?; + for entry in entries.filter_map(Result::ok) { + let path = entry.path(); + if path.is_dir() && !current_dir.starts_with(&path) && path != current_dir { + let _ = fs::remove_dir_all(&path); + } + } + Ok(()) +} + +fn extract_resource(compressed: &[u8], dest: &Path) -> Result<(), String> { + if compressed.is_empty() { + return Err(format!("embedded resource is empty: {}", dest.display())); + } + + let decompressed = + zstd::decode_all(compressed).map_err(|e| format!("decompress {}: {e}", dest.display()))?; + fs::write(dest, decompressed).map_err(|e| format!("write {}: {e}", dest.display())) +} diff --git a/crates/openshell-driver-vm/src/ffi.rs b/crates/openshell-driver-vm/src/ffi.rs new file mode 100644 index 0000000000..750788ac13 --- /dev/null +++ b/crates/openshell-driver-vm/src/ffi.rs @@ -0,0 +1,206 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Minimal runtime-loaded bindings for the libkrun C API used by the VM driver. + +#![allow(unsafe_code)] + +use std::ffi::{CStr, CString}; +use std::path::{Path, PathBuf}; +use std::sync::OnceLock; + +use libc::c_char; +use libloading::Library; + +use crate::runtime::validate_runtime_dir; + +pub const KRUN_LOG_TARGET_DEFAULT: i32 = -1; +pub const KRUN_LOG_LEVEL_OFF: u32 = 0; +pub const KRUN_LOG_LEVEL_ERROR: u32 = 1; +pub const KRUN_LOG_LEVEL_WARN: u32 = 2; +pub const KRUN_LOG_LEVEL_INFO: u32 = 3; +pub const KRUN_LOG_LEVEL_DEBUG: u32 = 4; +pub const KRUN_LOG_LEVEL_TRACE: u32 = 5; +pub const KRUN_LOG_STYLE_AUTO: u32 = 0; +pub const KRUN_LOG_OPTION_NO_ENV: u32 = 1; + +type KrunInitLog = + unsafe extern "C" fn(target_fd: i32, level: u32, style: u32, options: u32) -> i32; +type KrunCreateCtx = unsafe extern "C" fn() -> i32; +type KrunFreeCtx = unsafe extern "C" fn(ctx_id: u32) -> i32; +type KrunSetVmConfig = unsafe extern "C" fn(ctx_id: u32, num_vcpus: u8, ram_mib: u32) -> i32; +type KrunSetRoot = unsafe extern "C" fn(ctx_id: u32, root_path: *const c_char) -> i32; +type KrunSetWorkdir = unsafe extern "C" fn(ctx_id: u32, workdir_path: *const c_char) -> i32; +type KrunSetExec = unsafe extern "C" fn( + ctx_id: u32, + exec_path: *const c_char, + argv: *const *const c_char, + envp: *const *const c_char, +) -> i32; +type KrunSetPortMap = unsafe extern "C" fn(ctx_id: u32, port_map: *const *const c_char) -> i32; +type KrunSetConsoleOutput = unsafe extern "C" fn(ctx_id: u32, filepath: *const c_char) -> i32; +type KrunStartEnter = unsafe extern "C" fn(ctx_id: u32) -> i32; +type KrunDisableImplicitVsock = unsafe extern "C" fn(ctx_id: u32) -> i32; +type KrunAddVsock = unsafe extern "C" fn(ctx_id: u32, tsi_features: u32) -> i32; +#[cfg(target_os = "macos")] +type KrunAddNetUnixgram = unsafe extern "C" fn( + ctx_id: u32, + c_path: *const c_char, + fd: i32, + c_mac: *const u8, + features: u32, + flags: u32, +) -> i32; +type KrunAddNetUnixstream = unsafe extern "C" fn( + ctx_id: u32, + c_path: *const c_char, + fd: i32, + c_mac: *const u8, + features: u32, + flags: u32, +) -> i32; + +pub struct LibKrun { + pub krun_init_log: KrunInitLog, + pub krun_create_ctx: KrunCreateCtx, + pub krun_free_ctx: KrunFreeCtx, + pub krun_set_vm_config: KrunSetVmConfig, + pub krun_set_root: KrunSetRoot, + pub krun_set_workdir: KrunSetWorkdir, + pub krun_set_exec: KrunSetExec, + pub krun_set_port_map: KrunSetPortMap, + pub krun_set_console_output: KrunSetConsoleOutput, + pub krun_start_enter: KrunStartEnter, + pub krun_disable_implicit_vsock: KrunDisableImplicitVsock, + pub krun_add_vsock: KrunAddVsock, + #[cfg(target_os = "macos")] + pub krun_add_net_unixgram: KrunAddNetUnixgram, + #[allow(dead_code)] // Used on Linux when gvproxy runs in qemu/unixstream mode. + pub krun_add_net_unixstream: KrunAddNetUnixstream, +} + +static LIBKRUN: OnceLock = OnceLock::new(); + +pub fn libkrun(runtime_dir: &Path) -> Result<&'static LibKrun, String> { + if let Some(lib) = LIBKRUN.get() { + return Ok(lib); + } + + validate_runtime_dir(runtime_dir)?; + let loaded = LibKrun::load(runtime_dir)?; + let _ = LIBKRUN.set(loaded); + Ok(LIBKRUN.get().expect("libkrun should be initialized")) +} + +pub fn required_runtime_lib_name() -> &'static str { + #[cfg(target_os = "macos")] + { + "libkrun.dylib" + } + #[cfg(not(target_os = "macos"))] + { + "libkrun.so" + } +} + +impl LibKrun { + fn load(runtime_dir: &Path) -> Result { + let libkrun_path = runtime_dir.join(required_runtime_lib_name()); + preload_runtime_support_libraries(runtime_dir)?; + + let library = Box::leak(Box::new(unsafe { + Library::new(&libkrun_path) + .map_err(|e| format!("load libkrun from {}: {e}", libkrun_path.display()))? + })); + + Ok(Self { + krun_init_log: load_symbol(library, b"krun_init_log\0", &libkrun_path)?, + krun_create_ctx: load_symbol(library, b"krun_create_ctx\0", &libkrun_path)?, + krun_free_ctx: load_symbol(library, b"krun_free_ctx\0", &libkrun_path)?, + krun_set_vm_config: load_symbol(library, b"krun_set_vm_config\0", &libkrun_path)?, + krun_set_root: load_symbol(library, b"krun_set_root\0", &libkrun_path)?, + krun_set_workdir: load_symbol(library, b"krun_set_workdir\0", &libkrun_path)?, + krun_set_exec: load_symbol(library, b"krun_set_exec\0", &libkrun_path)?, + krun_set_port_map: load_symbol(library, b"krun_set_port_map\0", &libkrun_path)?, + krun_set_console_output: load_symbol( + library, + b"krun_set_console_output\0", + &libkrun_path, + )?, + krun_start_enter: load_symbol(library, b"krun_start_enter\0", &libkrun_path)?, + krun_disable_implicit_vsock: load_symbol( + library, + b"krun_disable_implicit_vsock\0", + &libkrun_path, + )?, + krun_add_vsock: load_symbol(library, b"krun_add_vsock\0", &libkrun_path)?, + #[cfg(target_os = "macos")] + krun_add_net_unixgram: load_symbol(library, b"krun_add_net_unixgram\0", &libkrun_path)?, + krun_add_net_unixstream: load_symbol( + library, + b"krun_add_net_unixstream\0", + &libkrun_path, + )?, + }) + } +} + +fn preload_runtime_support_libraries(runtime_dir: &Path) -> Result, String> { + let entries = std::fs::read_dir(runtime_dir) + .map_err(|e| format!("read {}: {e}", runtime_dir.display()))?; + + let mut support_libs: Vec = entries + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| { + #[cfg(target_os = "macos")] + { + name.starts_with("libkrunfw") && name.ends_with(".dylib") + } + #[cfg(not(target_os = "macos"))] + { + name.starts_with("libkrunfw") && name.contains(".so") + } + }) + }) + .collect(); + + support_libs.sort(); + for path in &support_libs { + let path_cstr = CString::new(path.to_string_lossy().as_bytes()) + .map_err(|e| format!("invalid support library path {}: {e}", path.display()))?; + let handle = + unsafe { libc::dlopen(path_cstr.as_ptr(), libc::RTLD_NOW | libc::RTLD_GLOBAL) }; + if handle.is_null() { + let error = unsafe { + let err = libc::dlerror(); + if err.is_null() { + "unknown dlopen error".to_string() + } else { + CStr::from_ptr(err).to_string_lossy().into_owned() + } + }; + return Err(format!( + "preload runtime support library {}: {error}", + path.display() + )); + } + } + + Ok(support_libs) +} + +fn load_symbol(library: &'static Library, name: &[u8], path: &Path) -> Result { + unsafe { + library.get::(name).map(|symbol| *symbol).map_err(|e| { + format!( + "load symbol {} from {}: {e}", + String::from_utf8_lossy(name).trim_end_matches('\0'), + path.display() + ) + }) + } +} diff --git a/crates/openshell-driver-vm/src/lib.rs b/crates/openshell-driver-vm/src/lib.rs new file mode 100644 index 0000000000..1c424deebe --- /dev/null +++ b/crates/openshell-driver-vm/src/lib.rs @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +pub mod driver; +mod embedded_runtime; +mod ffi; +mod rootfs; +mod runtime; + +pub const GUEST_SSH_PORT: u16 = 2222; + +pub use driver::{VmDriver, VmDriverConfig}; +pub use runtime::{VM_RUNTIME_DIR_ENV, VmLaunchConfig, configured_runtime_dir, run_vm}; diff --git a/crates/openshell-driver-vm/src/main.rs b/crates/openshell-driver-vm/src/main.rs new file mode 100644 index 0000000000..3a7976273d --- /dev/null +++ b/crates/openshell-driver-vm/src/main.rs @@ -0,0 +1,229 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use clap::Parser; +use miette::{IntoDiagnostic, Result}; +use openshell_core::VERSION; +use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; +use openshell_driver_vm::{ + VM_RUNTIME_DIR_ENV, VmDriver, VmDriverConfig, VmLaunchConfig, configured_runtime_dir, run_vm, +}; +use std::net::SocketAddr; +use std::path::PathBuf; +use tokio::net::UnixListener; +use tokio_stream::wrappers::UnixListenerStream; +use tracing::info; +use tracing_subscriber::EnvFilter; + +#[derive(Parser, Debug)] +#[command(name = "openshell-driver-vm")] +#[command(version = VERSION)] +struct Args { + #[arg(long, hide = true, default_value_t = false)] + internal_run_vm: bool, + + #[arg(long, hide = true)] + vm_rootfs: Option, + + #[arg(long, hide = true)] + vm_exec: Option, + + #[arg(long, hide = true, default_value = "/")] + vm_workdir: String, + + #[arg(long, hide = true)] + vm_env: Vec, + + #[arg(long, hide = true)] + vm_port: Vec, + + #[arg(long, hide = true)] + vm_console_output: Option, + + #[arg(long, hide = true, default_value_t = 2)] + vm_vcpus: u8, + + #[arg(long, hide = true, default_value_t = 2048)] + vm_mem_mib: u32, + + #[arg(long, hide = true, default_value_t = 1)] + vm_krun_log_level: u32, + + #[arg( + long, + env = "OPENSHELL_COMPUTE_DRIVER_BIND", + default_value = "127.0.0.1:50061" + )] + bind_address: SocketAddr, + + #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_SOCKET")] + bind_socket: Option, + + #[arg(long, env = "OPENSHELL_LOG_LEVEL", default_value = "info")] + log_level: String, + + #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] + openshell_endpoint: Option, + + #[arg( + long, + env = "OPENSHELL_VM_DRIVER_STATE_DIR", + default_value = "target/openshell-vm-driver" + )] + state_dir: PathBuf, + + #[arg(long, env = "OPENSHELL_SSH_HANDSHAKE_SECRET")] + ssh_handshake_secret: Option, + + #[arg(long, env = "OPENSHELL_SSH_HANDSHAKE_SKEW_SECS", default_value_t = 300)] + ssh_handshake_skew_secs: u64, + + #[arg(long = "guest-tls-ca", env = "OPENSHELL_VM_TLS_CA")] + guest_tls_ca: Option, + + #[arg(long = "guest-tls-cert", env = "OPENSHELL_VM_TLS_CERT")] + guest_tls_cert: Option, + + #[arg(long = "guest-tls-key", env = "OPENSHELL_VM_TLS_KEY")] + guest_tls_key: Option, + + #[arg(long, env = "OPENSHELL_VM_KRUN_LOG_LEVEL", default_value_t = 1)] + krun_log_level: u32, + + #[arg(long, env = "OPENSHELL_VM_DRIVER_VCPUS", default_value_t = 2)] + vcpus: u8, + + #[arg(long, env = "OPENSHELL_VM_DRIVER_MEM_MIB", default_value_t = 2048)] + mem_mib: u32, +} + +#[tokio::main] +async fn main() -> Result<()> { + let args = Args::parse(); + if args.internal_run_vm { + maybe_reexec_internal_vm_with_runtime_env()?; + let config = build_vm_launch_config(&args).map_err(|err| miette::miette!("{err}"))?; + run_vm(&config).map_err(|err| miette::miette!("{err}"))?; + return Ok(()); + } + + tracing_subscriber::fmt() + .with_env_filter( + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)), + ) + .init(); + + let driver = VmDriver::new(VmDriverConfig { + openshell_endpoint: args + .openshell_endpoint + .ok_or_else(|| miette::miette!("OPENSHELL_GRPC_ENDPOINT is required"))?, + state_dir: args.state_dir, + launcher_bin: None, + ssh_handshake_secret: args.ssh_handshake_secret.unwrap_or_default(), + ssh_handshake_skew_secs: args.ssh_handshake_skew_secs, + log_level: args.log_level, + krun_log_level: args.krun_log_level, + vcpus: args.vcpus, + mem_mib: args.mem_mib, + guest_tls_ca: args.guest_tls_ca, + guest_tls_cert: args.guest_tls_cert, + guest_tls_key: args.guest_tls_key, + }) + .await + .map_err(|err| miette::miette!("{err}"))?; + + if let Some(socket_path) = args.bind_socket { + if let Some(parent) = socket_path.parent() { + std::fs::create_dir_all(parent).into_diagnostic()?; + } + match std::fs::remove_file(&socket_path) { + Ok(()) => {} + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + Err(err) => return Err(err).into_diagnostic(), + } + + info!(socket = %socket_path.display(), "Starting vm compute driver"); + let listener = UnixListener::bind(&socket_path).into_diagnostic()?; + let result = tonic::transport::Server::builder() + .add_service(ComputeDriverServer::new(driver)) + .serve_with_incoming(UnixListenerStream::new(listener)) + .await + .into_diagnostic(); + let _ = std::fs::remove_file(&socket_path); + result + } else { + info!(address = %args.bind_address, "Starting vm compute driver"); + tonic::transport::Server::builder() + .add_service(ComputeDriverServer::new(driver)) + .serve(args.bind_address) + .await + .into_diagnostic() + } +} + +fn build_vm_launch_config(args: &Args) -> std::result::Result { + let rootfs = args + .vm_rootfs + .clone() + .ok_or_else(|| "--vm-rootfs is required in internal VM mode".to_string())?; + let exec_path = args + .vm_exec + .clone() + .ok_or_else(|| "--vm-exec is required in internal VM mode".to_string())?; + let console_output = args + .vm_console_output + .clone() + .ok_or_else(|| "--vm-console-output is required in internal VM mode".to_string())?; + + Ok(VmLaunchConfig { + rootfs, + vcpus: args.vm_vcpus, + mem_mib: args.vm_mem_mib, + exec_path, + args: Vec::new(), + env: args.vm_env.clone(), + workdir: args.vm_workdir.clone(), + port_map: args.vm_port.clone(), + log_level: args.vm_krun_log_level, + console_output, + }) +} + +#[cfg(target_os = "macos")] +fn maybe_reexec_internal_vm_with_runtime_env() -> Result<()> { + const REEXEC_ENV: &str = "__OPENSHELL_DRIVER_VM_REEXEC"; + + if std::env::var_os(REEXEC_ENV).is_some() { + return Ok(()); + } + + let runtime_dir = configured_runtime_dir().map_err(|err| miette::miette!("{err}"))?; + let runtime_str = runtime_dir.to_string_lossy(); + let needs_reexec = std::env::var_os("DYLD_LIBRARY_PATH") + .is_none_or(|value| !value.to_string_lossy().contains(runtime_str.as_ref())); + if !needs_reexec { + return Ok(()); + } + + let mut dyld_paths = vec![runtime_dir.clone()]; + if let Some(existing) = std::env::var_os("DYLD_LIBRARY_PATH") { + dyld_paths.extend(std::env::split_paths(&existing)); + } + let joined = std::env::join_paths(&dyld_paths) + .map_err(|err| miette::miette!("join DYLD_LIBRARY_PATH: {err}"))?; + let exe = std::env::current_exe().into_diagnostic()?; + let args: Vec = std::env::args().skip(1).collect(); + let status = std::process::Command::new(exe) + .args(&args) + .env("DYLD_LIBRARY_PATH", &joined) + .env(VM_RUNTIME_DIR_ENV, runtime_dir) + .env(REEXEC_ENV, "1") + .status() + .into_diagnostic()?; + std::process::exit(status.code().unwrap_or(1)); +} + +#[cfg(not(target_os = "macos"))] +fn maybe_reexec_internal_vm_with_runtime_env() -> Result<()> { + Ok(()) +} diff --git a/crates/openshell-driver-vm/src/rootfs.rs b/crates/openshell-driver-vm/src/rootfs.rs new file mode 100644 index 0000000000..b9b29b5fc0 --- /dev/null +++ b/crates/openshell-driver-vm/src/rootfs.rs @@ -0,0 +1,234 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::fs; +use std::io::Cursor; +use std::path::Path; + +const ROOTFS: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/rootfs.tar.zst")); +const ROOTFS_VARIANT_MARKER: &str = ".openshell-rootfs-variant"; +const SANDBOX_GUEST_INIT_PATH: &str = "/srv/openshell-vm-sandbox-init.sh"; + +pub const fn sandbox_guest_init_path() -> &'static str { + SANDBOX_GUEST_INIT_PATH +} + +pub fn extract_sandbox_rootfs_to(dest: &Path) -> Result<(), String> { + if ROOTFS.is_empty() { + return Err( + "sandbox rootfs not embedded. Build openshell-driver-vm with OPENSHELL_VM_RUNTIME_COMPRESSED_DIR set or run `mise run vm:setup` first" + .to_string(), + ); + } + + let expected_marker = format!("{}:sandbox", env!("CARGO_PKG_VERSION")); + let marker_path = dest.join(ROOTFS_VARIANT_MARKER); + + if dest.is_dir() + && fs::read_to_string(&marker_path) + .map(|value| value.trim() == expected_marker) + .unwrap_or(false) + { + return Ok(()); + } + + if dest.exists() { + fs::remove_dir_all(dest) + .map_err(|e| format!("remove old rootfs {}: {e}", dest.display()))?; + } + + extract_rootfs_to(dest)?; + prepare_sandbox_rootfs(dest)?; + fs::write(marker_path, format!("{expected_marker}\n")) + .map_err(|e| format!("write rootfs variant marker: {e}"))?; + Ok(()) +} + +fn extract_rootfs_to(dest: &Path) -> Result<(), String> { + fs::create_dir_all(dest).map_err(|e| format!("create rootfs dir {}: {e}", dest.display()))?; + + let decoder = + zstd::Decoder::new(Cursor::new(ROOTFS)).map_err(|e| format!("decompress rootfs: {e}"))?; + let mut archive = tar::Archive::new(decoder); + archive + .unpack(dest) + .map_err(|e| format!("extract rootfs tarball into {}: {e}", dest.display())) +} + +fn prepare_sandbox_rootfs(rootfs: &Path) -> Result<(), String> { + for relative in [ + "usr/local/bin/k3s", + "usr/local/bin/kubectl", + "var/lib/rancher", + "etc/rancher", + "opt/openshell/charts", + "opt/openshell/manifests", + "opt/openshell/.initialized", + "opt/openshell/.rootfs-type", + ] { + remove_rootfs_path(rootfs, relative)?; + } + + let init_path = rootfs.join("srv/openshell-vm-sandbox-init.sh"); + if let Some(parent) = init_path.parent() { + fs::create_dir_all(parent).map_err(|e| format!("create {}: {e}", parent.display()))?; + } + fs::write( + &init_path, + include_str!("../scripts/openshell-vm-sandbox-init.sh"), + ) + .map_err(|e| format!("write {}: {e}", init_path.display()))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + + fs::set_permissions(&init_path, fs::Permissions::from_mode(0o755)) + .map_err(|e| format!("chmod {}: {e}", init_path.display()))?; + } + + let opt_dir = rootfs.join("opt/openshell"); + fs::create_dir_all(&opt_dir).map_err(|e| format!("create {}: {e}", opt_dir.display()))?; + fs::write(opt_dir.join(".rootfs-type"), "sandbox\n") + .map_err(|e| format!("write sandbox rootfs marker: {e}"))?; + ensure_sandbox_guest_user(rootfs)?; + fs::create_dir_all(rootfs.join("sandbox")) + .map_err(|e| format!("create sandbox workdir: {e}"))?; + + Ok(()) +} + +fn ensure_sandbox_guest_user(rootfs: &Path) -> Result<(), String> { + const SANDBOX_UID: u32 = 10001; + const SANDBOX_GID: u32 = 10001; + + let etc_dir = rootfs.join("etc"); + fs::create_dir_all(&etc_dir).map_err(|e| format!("create {}: {e}", etc_dir.display()))?; + + ensure_line_in_file( + &etc_dir.join("group"), + &format!("sandbox:x:{SANDBOX_GID}:"), + |line| line.starts_with("sandbox:"), + )?; + ensure_line_in_file(&etc_dir.join("gshadow"), "sandbox:!::", |line| { + line.starts_with("sandbox:") + })?; + ensure_line_in_file( + &etc_dir.join("passwd"), + &format!("sandbox:x:{SANDBOX_UID}:{SANDBOX_GID}:OpenShell Sandbox:/sandbox:/bin/bash"), + |line| line.starts_with("sandbox:"), + )?; + ensure_line_in_file( + &etc_dir.join("shadow"), + "sandbox:!:20123:0:99999:7:::", + |line| line.starts_with("sandbox:"), + )?; + + Ok(()) +} + +fn ensure_line_in_file( + path: &Path, + line: &str, + exists: impl Fn(&str) -> bool, +) -> Result<(), String> { + let mut contents = if path.exists() { + fs::read_to_string(path).map_err(|e| format!("read {}: {e}", path.display()))? + } else { + String::new() + }; + + if contents.lines().any(exists) { + return Ok(()); + } + + if !contents.is_empty() && !contents.ends_with('\n') { + contents.push('\n'); + } + contents.push_str(line); + contents.push('\n'); + + fs::write(path, contents).map_err(|e| format!("write {}: {e}", path.display())) +} + +fn remove_rootfs_path(rootfs: &Path, relative: &str) -> Result<(), String> { + let path = rootfs.join(relative); + if !path.exists() { + return Ok(()); + } + + let result = if path.is_dir() { + fs::remove_dir_all(&path) + } else { + fs::remove_file(&path) + }; + result.map_err(|e| format!("remove {}: {e}", path.display())) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::PathBuf; + use std::sync::atomic::{AtomicU64, Ordering}; + use std::time::{SystemTime, UNIX_EPOCH}; + + #[test] + fn prepare_sandbox_rootfs_rewrites_guest_layout() { + let dir = unique_temp_dir(); + let rootfs = dir.join("rootfs"); + + fs::create_dir_all(rootfs.join("usr/local/bin")).expect("create usr/local/bin"); + fs::create_dir_all(rootfs.join("etc")).expect("create etc"); + fs::create_dir_all(rootfs.join("var/lib/rancher")).expect("create var/lib/rancher"); + fs::create_dir_all(rootfs.join("opt/openshell/charts")).expect("create charts"); + fs::create_dir_all(rootfs.join("opt/openshell/manifests")).expect("create manifests"); + fs::write(rootfs.join("usr/local/bin/k3s"), b"k3s").expect("write k3s"); + fs::write(rootfs.join("usr/local/bin/kubectl"), b"kubectl").expect("write kubectl"); + fs::write(rootfs.join("opt/openshell/.initialized"), b"yes").expect("write initialized"); + fs::write( + rootfs.join("etc/passwd"), + "root:x:0:0:root:/root:/bin/bash\n", + ) + .expect("write passwd"); + fs::write(rootfs.join("etc/group"), "root:x:0:\n").expect("write group"); + fs::write(rootfs.join("etc/hosts"), "127.0.0.1 localhost\n").expect("write hosts"); + + prepare_sandbox_rootfs(&rootfs).expect("prepare sandbox rootfs"); + + assert!(!rootfs.join("usr/local/bin/k3s").exists()); + assert!(!rootfs.join("usr/local/bin/kubectl").exists()); + assert!(!rootfs.join("var/lib/rancher").exists()); + assert!(!rootfs.join("opt/openshell/charts").exists()); + assert!(!rootfs.join("opt/openshell/manifests").exists()); + assert!(rootfs.join("srv/openshell-vm-sandbox-init.sh").is_file()); + assert!(rootfs.join("sandbox").is_dir()); + assert!( + fs::read_to_string(rootfs.join("etc/passwd")) + .expect("read passwd") + .contains("sandbox:x:10001:10001:OpenShell Sandbox:/sandbox:/bin/bash") + ); + assert!( + fs::read_to_string(rootfs.join("etc/group")) + .expect("read group") + .contains("sandbox:x:10001:") + ); + assert_eq!( + fs::read_to_string(rootfs.join("etc/hosts")).expect("read hosts"), + "127.0.0.1 localhost\n" + ); + + let _ = fs::remove_dir_all(&dir); + } + + fn unique_temp_dir() -> PathBuf { + static COUNTER: AtomicU64 = AtomicU64::new(0); + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("time went backwards") + .as_nanos(); + let suffix = COUNTER.fetch_add(1, Ordering::Relaxed); + std::env::temp_dir().join(format!( + "openshell-driver-vm-rootfs-test-{}-{nanos}-{suffix}", + std::process::id() + )) + } +} diff --git a/crates/openshell-driver-vm/src/runtime.rs b/crates/openshell-driver-vm/src/runtime.rs new file mode 100644 index 0000000000..9888feb182 --- /dev/null +++ b/crates/openshell-driver-vm/src/runtime.rs @@ -0,0 +1,877 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#![allow(unsafe_code)] + +use std::ffi::CString; +use std::io::{Read, Write}; +use std::os::unix::net::UnixStream; +use std::path::{Path, PathBuf}; +use std::process::{Child as StdChild, Command as StdCommand, Stdio}; +use std::ptr; +use std::sync::atomic::{AtomicI32, Ordering}; +use std::time::{Duration, Instant}; + +use crate::{GUEST_SSH_PORT, embedded_runtime, ffi}; + +pub const VM_RUNTIME_DIR_ENV: &str = "OPENSHELL_VM_RUNTIME_DIR"; + +static CHILD_PID: AtomicI32 = AtomicI32::new(0); + +pub struct VmLaunchConfig { + pub rootfs: PathBuf, + pub vcpus: u8, + pub mem_mib: u32, + pub exec_path: String, + pub args: Vec, + pub env: Vec, + pub workdir: String, + pub port_map: Vec, + pub log_level: u32, + pub console_output: PathBuf, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct PortMapping { + host_port: u16, + guest_port: u16, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct GvproxyPortPlan { + ssh_port: u16, + forwarded_ports: Vec, +} + +pub fn run_vm(config: &VmLaunchConfig) -> Result<(), String> { + if !config.rootfs.is_dir() { + return Err(format!( + "rootfs directory not found: {}", + config.rootfs.display() + )); + } + + #[cfg(target_os = "linux")] + check_kvm_access()?; + + let runtime_dir = configured_runtime_dir()?; + validate_runtime_dir(&runtime_dir)?; + configure_runtime_loader_env(&runtime_dir)?; + raise_nofile_limit(); + + let vm = VmContext::create(&runtime_dir, config.log_level)?; + vm.set_vm_config(config.vcpus, config.mem_mib)?; + vm.set_root(&config.rootfs)?; + vm.set_workdir(&config.workdir)?; + + let mut forwarded_port_map = config.port_map.clone(); + let mut gvproxy_guard = None; + let mut gvproxy_api_sock = None; + if !config.port_map.is_empty() { + let gvproxy_binary = runtime_dir.join("gvproxy"); + if !gvproxy_binary.is_file() { + return Err(format!( + "missing runtime file: {}", + gvproxy_binary.display() + )); + } + + kill_stale_gvproxy_by_port_map(&config.port_map); + + let sock_base = gvproxy_socket_base(&config.rootfs)?; + let net_sock = sock_base.with_extension("v"); + let api_sock = sock_base.with_extension("a"); + let _ = std::fs::remove_file(&net_sock); + let _ = std::fs::remove_file(&api_sock); + let _ = std::fs::remove_file(sock_base.with_extension("v-krun.sock")); + + let run_dir = config.rootfs.parent().unwrap_or(&config.rootfs); + let gvproxy_log = run_dir.join("gvproxy.log"); + let gvproxy_log_file = std::fs::File::create(&gvproxy_log) + .map_err(|e| format!("create gvproxy log {}: {e}", gvproxy_log.display()))?; + + let gvproxy_ports = plan_gvproxy_ports(&config.port_map)?; + forwarded_port_map = gvproxy_ports.forwarded_ports; + + #[cfg(target_os = "linux")] + let (gvproxy_net_flag, gvproxy_net_url) = + ("-listen-qemu", format!("unix://{}", net_sock.display())); + #[cfg(target_os = "macos")] + let (gvproxy_net_flag, gvproxy_net_url) = ( + "-listen-vfkit", + format!("unixgram://{}", net_sock.display()), + ); + + let child = StdCommand::new(&gvproxy_binary) + .arg(gvproxy_net_flag) + .arg(&gvproxy_net_url) + .arg("-listen") + .arg(format!("unix://{}", api_sock.display())) + .arg("-ssh-port") + .arg(gvproxy_ports.ssh_port.to_string()) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(gvproxy_log_file) + .spawn() + .map_err(|e| format!("failed to start gvproxy {}: {e}", gvproxy_binary.display()))?; + + wait_for_path(&net_sock, Duration::from_secs(5), "gvproxy data socket")?; + + vm.disable_implicit_vsock()?; + vm.add_vsock(0)?; + + let mac: [u8; 6] = [0x5a, 0x94, 0xef, 0xe4, 0x0c, 0xee]; + const NET_FEATURE_CSUM: u32 = 1 << 0; + const NET_FEATURE_GUEST_CSUM: u32 = 1 << 1; + const NET_FEATURE_GUEST_TSO4: u32 = 1 << 7; + const NET_FEATURE_GUEST_UFO: u32 = 1 << 10; + const NET_FEATURE_HOST_TSO4: u32 = 1 << 11; + const NET_FEATURE_HOST_UFO: u32 = 1 << 14; + const COMPAT_NET_FEATURES: u32 = NET_FEATURE_CSUM + | NET_FEATURE_GUEST_CSUM + | NET_FEATURE_GUEST_TSO4 + | NET_FEATURE_GUEST_UFO + | NET_FEATURE_HOST_TSO4 + | NET_FEATURE_HOST_UFO; + + #[cfg(target_os = "linux")] + vm.add_net_unixstream(&net_sock, &mac, COMPAT_NET_FEATURES)?; + #[cfg(target_os = "macos")] + { + const NET_FLAG_VFKIT: u32 = 1 << 0; + vm.add_net_unixgram(&net_sock, &mac, COMPAT_NET_FEATURES, NET_FLAG_VFKIT)?; + } + + gvproxy_guard = Some(GvproxyGuard::new(child)); + gvproxy_api_sock = Some(api_sock); + } + + if !config.port_map.is_empty() && gvproxy_api_sock.is_none() { + vm.set_port_map(&config.port_map)?; + } + vm.set_console_output(&config.console_output)?; + + let env = if config.env.is_empty() { + vec![ + "HOME=/root".to_string(), + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin".to_string(), + "TERM=xterm".to_string(), + ] + } else { + config.env.clone() + }; + vm.set_exec(&config.exec_path, &config.args, &env)?; + + let pid = unsafe { libc::fork() }; + match pid { + -1 => Err(format!("fork failed: {}", std::io::Error::last_os_error())), + 0 => { + let ret = vm.start_enter(); + eprintln!("krun_start_enter failed: {ret}"); + std::process::exit(1); + } + _ => { + install_signal_forwarding(pid); + + let port_forward_result = if let Some(api_sock) = gvproxy_api_sock.as_ref() { + expose_port_map(api_sock, &forwarded_port_map) + } else { + Ok(()) + }; + + if let Err(err) = port_forward_result { + unsafe { + libc::kill(pid, libc::SIGTERM); + } + let _ = wait_for_child(pid); + cleanup_gvproxy(gvproxy_guard); + return Err(err); + } + + let status = wait_for_child(pid)?; + CHILD_PID.store(0, Ordering::Relaxed); + cleanup_gvproxy(gvproxy_guard); + + if libc::WIFEXITED(status) { + match libc::WEXITSTATUS(status) { + 0 => Ok(()), + code => Err(format!("VM exited with status {code}")), + } + } else if libc::WIFSIGNALED(status) { + let sig = libc::WTERMSIG(status); + Err(format!("VM killed by signal {sig}")) + } else { + Err(format!("VM exited with unexpected wait status {status}")) + } + } + } +} + +pub fn validate_runtime_dir(dir: &Path) -> Result<(), String> { + if !dir.is_dir() { + return Err(format!( + "VM runtime not found at {}. Run `mise run vm:setup` or set {VM_RUNTIME_DIR_ENV}", + dir.display() + )); + } + + embedded_runtime::validate_runtime_dir(dir) +} + +pub fn configured_runtime_dir() -> Result { + if let Some(path) = std::env::var_os(VM_RUNTIME_DIR_ENV) { + return Ok(PathBuf::from(path)); + } + embedded_runtime::ensure_runtime_extracted() +} + +#[cfg(target_os = "macos")] +fn configure_runtime_loader_env(runtime_dir: &Path) -> Result<(), String> { + let existing = std::env::var_os("DYLD_FALLBACK_LIBRARY_PATH"); + let mut paths = vec![runtime_dir.to_path_buf()]; + if let Some(existing) = existing { + paths.extend(std::env::split_paths(&existing)); + } + let joined = + std::env::join_paths(paths).map_err(|e| format!("join DYLD_FALLBACK_LIBRARY_PATH: {e}"))?; + unsafe { + std::env::set_var("DYLD_FALLBACK_LIBRARY_PATH", joined); + } + Ok(()) +} + +#[cfg(target_os = "linux")] +fn configure_runtime_loader_env(runtime_dir: &Path) -> Result<(), String> { + let existing = std::env::var_os("LD_LIBRARY_PATH"); + let mut paths = vec![runtime_dir.to_path_buf()]; + if let Some(existing) = existing { + paths.extend(std::env::split_paths(&existing)); + } + let joined = std::env::join_paths(paths).map_err(|e| format!("join LD_LIBRARY_PATH: {e}"))?; + unsafe { + std::env::set_var("LD_LIBRARY_PATH", joined); + } + Ok(()) +} + +#[cfg(not(any(target_os = "macos", target_os = "linux")))] +fn configure_runtime_loader_env(_runtime_dir: &Path) -> Result<(), String> { + Ok(()) +} + +fn raise_nofile_limit() { + #[cfg(unix)] + unsafe { + let mut rlim = libc::rlimit { + rlim_cur: 0, + rlim_max: 0, + }; + if libc::getrlimit(libc::RLIMIT_NOFILE, &raw mut rlim) == 0 { + rlim.rlim_cur = rlim.rlim_max; + let _ = libc::setrlimit(libc::RLIMIT_NOFILE, &raw const rlim); + } + } +} + +fn clamp_log_level(level: u32) -> u32 { + match level { + 0 => ffi::KRUN_LOG_LEVEL_OFF, + 1 => ffi::KRUN_LOG_LEVEL_ERROR, + 2 => ffi::KRUN_LOG_LEVEL_WARN, + 3 => ffi::KRUN_LOG_LEVEL_INFO, + 4 => ffi::KRUN_LOG_LEVEL_DEBUG, + _ => ffi::KRUN_LOG_LEVEL_TRACE, + } +} + +struct VmContext { + krun: &'static ffi::LibKrun, + ctx_id: u32, +} + +impl VmContext { + fn create(runtime_dir: &Path, log_level: u32) -> Result { + let krun = ffi::libkrun(runtime_dir)?; + check( + unsafe { + (krun.krun_init_log)( + ffi::KRUN_LOG_TARGET_DEFAULT, + clamp_log_level(log_level), + ffi::KRUN_LOG_STYLE_AUTO, + ffi::KRUN_LOG_OPTION_NO_ENV, + ) + }, + "krun_init_log", + )?; + + let ctx_id = unsafe { (krun.krun_create_ctx)() }; + if ctx_id < 0 { + return Err(format!("krun_create_ctx failed with error code {ctx_id}")); + } + + Ok(Self { + krun, + ctx_id: ctx_id as u32, + }) + } + + fn set_vm_config(&self, vcpus: u8, mem_mib: u32) -> Result<(), String> { + check( + unsafe { (self.krun.krun_set_vm_config)(self.ctx_id, vcpus, mem_mib) }, + "krun_set_vm_config", + ) + } + + fn set_root(&self, rootfs: &Path) -> Result<(), String> { + let rootfs_c = path_to_cstring(rootfs)?; + check( + unsafe { (self.krun.krun_set_root)(self.ctx_id, rootfs_c.as_ptr()) }, + "krun_set_root", + ) + } + + fn set_workdir(&self, workdir: &str) -> Result<(), String> { + let workdir_c = CString::new(workdir).map_err(|e| format!("invalid workdir: {e}"))?; + check( + unsafe { (self.krun.krun_set_workdir)(self.ctx_id, workdir_c.as_ptr()) }, + "krun_set_workdir", + ) + } + + fn disable_implicit_vsock(&self) -> Result<(), String> { + check( + unsafe { (self.krun.krun_disable_implicit_vsock)(self.ctx_id) }, + "krun_disable_implicit_vsock", + ) + } + + fn add_vsock(&self, tsi_features: u32) -> Result<(), String> { + check( + unsafe { (self.krun.krun_add_vsock)(self.ctx_id, tsi_features) }, + "krun_add_vsock", + ) + } + + #[cfg(target_os = "macos")] + fn add_net_unixgram( + &self, + socket_path: &Path, + mac: &[u8; 6], + features: u32, + flags: u32, + ) -> Result<(), String> { + let sock_c = path_to_cstring(socket_path)?; + check( + unsafe { + (self.krun.krun_add_net_unixgram)( + self.ctx_id, + sock_c.as_ptr(), + -1, + mac.as_ptr(), + features, + flags, + ) + }, + "krun_add_net_unixgram", + ) + } + + #[allow(dead_code)] // Used on Linux when gvproxy runs in qemu/unixstream mode. + fn add_net_unixstream( + &self, + socket_path: &Path, + mac: &[u8; 6], + features: u32, + ) -> Result<(), String> { + let sock_c = path_to_cstring(socket_path)?; + check( + unsafe { + (self.krun.krun_add_net_unixstream)( + self.ctx_id, + sock_c.as_ptr(), + -1, + mac.as_ptr(), + features, + 0, + ) + }, + "krun_add_net_unixstream", + ) + } + + fn set_port_map(&self, port_map: &[String]) -> Result<(), String> { + let port_strs: Vec<&str> = port_map.iter().map(String::as_str).collect(); + let (_owners, ptrs) = c_string_array(&port_strs)?; + check( + unsafe { (self.krun.krun_set_port_map)(self.ctx_id, ptrs.as_ptr()) }, + "krun_set_port_map", + ) + } + + fn set_console_output(&self, path: &Path) -> Result<(), String> { + let console_c = path_to_cstring(path)?; + check( + unsafe { (self.krun.krun_set_console_output)(self.ctx_id, console_c.as_ptr()) }, + "krun_set_console_output", + ) + } + + fn set_exec(&self, exec_path: &str, args: &[String], env: &[String]) -> Result<(), String> { + let exec_c = CString::new(exec_path).map_err(|e| format!("invalid exec path: {e}"))?; + let argv_strs: Vec<&str> = args.iter().map(String::as_str).collect(); + let (_argv_owners, argv_ptrs) = c_string_array(&argv_strs)?; + let env_strs: Vec<&str> = env.iter().map(String::as_str).collect(); + let (_env_owners, env_ptrs) = c_string_array(&env_strs)?; + + check( + unsafe { + (self.krun.krun_set_exec)( + self.ctx_id, + exec_c.as_ptr(), + argv_ptrs.as_ptr(), + env_ptrs.as_ptr(), + ) + }, + "krun_set_exec", + ) + } + + fn start_enter(&self) -> i32 { + unsafe { (self.krun.krun_start_enter)(self.ctx_id) } + } +} + +impl Drop for VmContext { + fn drop(&mut self) { + let ret = unsafe { (self.krun.krun_free_ctx)(self.ctx_id) }; + if ret < 0 { + eprintln!( + "warning: krun_free_ctx({}) failed with code {ret}", + self.ctx_id + ); + } + } +} + +struct GvproxyGuard { + child: Option, +} + +impl GvproxyGuard { + fn new(child: StdChild) -> Self { + Self { child: Some(child) } + } + + fn disarm(&mut self) -> Option { + self.child.take() + } +} + +impl Drop for GvproxyGuard { + fn drop(&mut self) { + if let Some(mut child) = self.child.take() { + let _ = child.kill(); + let _ = child.wait(); + } + } +} + +fn expose_port_map(api_sock: &Path, port_map: &[String]) -> Result<(), String> { + wait_for_path(api_sock, Duration::from_secs(2), "gvproxy API socket")?; + let guest_ip = "192.168.127.2"; + + for pm in port_map { + let mapping = parse_port_mapping(pm)?; + + let expose_body = format!( + r#"{{"local":":{}","remote":"{guest_ip}:{}","protocol":"tcp"}}"#, + mapping.host_port, mapping.guest_port + ); + + let deadline = Instant::now() + Duration::from_secs(10); + let mut retry_interval = Duration::from_millis(100); + loop { + match gvproxy_expose(api_sock, &expose_body) { + Ok(()) => break, + Err(err) if Instant::now() < deadline => { + std::thread::sleep(retry_interval); + retry_interval = (retry_interval * 2).min(Duration::from_secs(1)); + if retry_interval == Duration::from_secs(1) { + eprintln!("retrying gvproxy port expose {pm}: {err}"); + } + } + Err(err) => { + return Err(format!( + "failed to forward port {} via gvproxy: {err}", + mapping.host_port + )); + } + } + } + } + + Ok(()) +} + +fn gvproxy_expose(api_sock: &Path, body: &str) -> Result<(), String> { + let mut stream = + UnixStream::connect(api_sock).map_err(|e| format!("connect to gvproxy API socket: {e}"))?; + + let request = format!( + "POST /services/forwarder/expose HTTP/1.1\r\n\ + Host: localhost\r\n\ + Content-Type: application/json\r\n\ + Content-Length: {}\r\n\ + Connection: close\r\n\ + \r\n\ + {}", + body.len(), + body, + ); + + stream + .write_all(request.as_bytes()) + .map_err(|e| format!("write to gvproxy API: {e}"))?; + + let mut buf = [0u8; 1024]; + let n = stream + .read(&mut buf) + .map_err(|e| format!("read from gvproxy API: {e}"))?; + let response = String::from_utf8_lossy(&buf[..n]); + let status = response + .lines() + .next() + .and_then(|line| line.split_whitespace().nth(1)) + .unwrap_or("0"); + + match status { + "200" | "204" => Ok(()), + _ => Err(format!( + "gvproxy API: {}", + response.lines().next().unwrap_or("") + )), + } +} + +fn plan_gvproxy_ports(port_map: &[String]) -> Result { + let mut ssh_port = None; + let mut forwarded_ports = Vec::with_capacity(port_map.len()); + + for pm in port_map { + let mapping = parse_port_mapping(pm)?; + if ssh_port.is_none() && mapping.guest_port == GUEST_SSH_PORT && mapping.host_port >= 1024 { + ssh_port = Some(mapping.host_port); + continue; + } + forwarded_ports.push(pm.clone()); + } + + Ok(GvproxyPortPlan { + ssh_port: match ssh_port { + Some(port) => port, + None => pick_gvproxy_ssh_port()?, + }, + forwarded_ports, + }) +} + +fn parse_port_mapping(pm: &str) -> Result { + let parts: Vec<&str> = pm.split(':').collect(); + let (host, guest) = match parts.as_slice() { + [host, guest] => (*host, *guest), + [port] => (*port, *port), + _ => return Err(format!("invalid port mapping '{pm}'")), + }; + + let host_port = host + .parse::() + .map_err(|_| format!("invalid port mapping '{pm}'"))?; + let guest_port = guest + .parse::() + .map_err(|_| format!("invalid port mapping '{pm}'"))?; + + Ok(PortMapping { + host_port, + guest_port, + }) +} + +fn wait_for_path(path: &Path, timeout: Duration, label: &str) -> Result<(), String> { + let deadline = Instant::now() + timeout; + let mut interval = Duration::from_millis(5); + while !path.exists() { + if Instant::now() >= deadline { + return Err(format!( + "{label} did not appear within {:.1}s: {}", + timeout.as_secs_f64(), + path.display() + )); + } + std::thread::sleep(interval); + interval = (interval * 2).min(Duration::from_millis(200)); + } + Ok(()) +} + +fn hash_path_id(path: &Path) -> String { + let mut hash: u64 = 0xcbf29ce484222325; + for byte in path.to_string_lossy().as_bytes() { + hash ^= u64::from(*byte); + hash = hash.wrapping_mul(0x100000001b3); + } + format!("{:012x}", hash & 0x0000_ffff_ffff_ffff) +} + +fn secure_socket_base(subdir: &str) -> Result { + let base = if let Some(xdg) = std::env::var_os("XDG_RUNTIME_DIR") { + PathBuf::from(xdg) + } else { + let mut base = PathBuf::from("/tmp"); + if !base.is_dir() { + base = std::env::temp_dir(); + } + base + }; + let dir = base.join(subdir); + + if dir.exists() { + let meta = dir + .symlink_metadata() + .map_err(|e| format!("lstat {}: {e}", dir.display()))?; + if meta.file_type().is_symlink() { + return Err(format!( + "socket directory {} is a symlink; refusing to use it", + dir.display() + )); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt as _; + let uid = unsafe { libc::getuid() }; + if meta.uid() != uid { + return Err(format!( + "socket directory {} is owned by uid {} but we are uid {}", + dir.display(), + meta.uid(), + uid + )); + } + } + } else { + std::fs::create_dir_all(&dir) + .map_err(|e| format!("create socket dir {}: {e}", dir.display()))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + let _ = std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)); + } + } + + Ok(dir) +} + +fn gvproxy_socket_base(rootfs: &Path) -> Result { + Ok(secure_socket_base("osd-gv")?.join(hash_path_id(rootfs))) +} + +fn pick_gvproxy_ssh_port() -> Result { + let listener = std::net::TcpListener::bind(("127.0.0.1", 0)) + .map_err(|e| format!("allocate gvproxy ssh port on localhost: {e}"))?; + let port = listener + .local_addr() + .map_err(|e| format!("read gvproxy ssh port: {e}"))? + .port(); + drop(listener); + Ok(port) +} + +fn kill_stale_gvproxy_by_port_map(port_map: &[String]) { + for pm in port_map { + if let Some(host_port) = pm + .split(':') + .next() + .and_then(|port| port.parse::().ok()) + { + kill_stale_gvproxy_by_port(host_port); + } + } +} + +fn kill_stale_gvproxy_by_port(port: u16) { + let output = StdCommand::new("lsof") + .args(["-ti", &format!(":{port}")]) + .output(); + + let pids = match output { + Ok(output) if output.status.success() => { + String::from_utf8_lossy(&output.stdout).to_string() + } + _ => return, + }; + + for line in pids.lines() { + if let Ok(pid) = line.trim().parse::() + && is_process_named(pid as libc::pid_t, "gvproxy") + { + kill_gvproxy_pid(pid); + } + } +} + +fn kill_gvproxy_pid(pid: u32) { + let pid = pid as libc::pid_t; + if unsafe { libc::kill(pid, 0) } != 0 { + return; + } + if !is_process_named(pid, "gvproxy") { + return; + } + unsafe { + libc::kill(pid, libc::SIGTERM); + } + std::thread::sleep(Duration::from_millis(200)); +} + +#[cfg(target_os = "macos")] +fn is_process_named(pid: libc::pid_t, expected: &str) -> bool { + StdCommand::new("ps") + .args(["-p", &pid.to_string(), "-o", "comm="]) + .output() + .ok() + .and_then(|output| { + if output.status.success() { + String::from_utf8(output.stdout).ok() + } else { + None + } + }) + .is_some_and(|name| name.trim().contains(expected)) +} + +#[cfg(target_os = "linux")] +fn is_process_named(pid: libc::pid_t, expected: &str) -> bool { + std::fs::read_to_string(format!("/proc/{pid}/comm")) + .map(|name| name.trim().contains(expected)) + .unwrap_or(false) +} + +#[cfg(not(any(target_os = "macos", target_os = "linux")))] +fn is_process_named(_pid: libc::pid_t, _expected: &str) -> bool { + false +} + +fn install_signal_forwarding(pid: i32) { + unsafe { + libc::signal( + libc::SIGINT, + forward_signal as *const () as libc::sighandler_t, + ); + libc::signal( + libc::SIGTERM, + forward_signal as *const () as libc::sighandler_t, + ); + } + CHILD_PID.store(pid, Ordering::Relaxed); +} + +extern "C" fn forward_signal(_sig: libc::c_int) { + let pid = CHILD_PID.load(Ordering::Relaxed); + if pid > 0 { + unsafe { + libc::kill(pid, libc::SIGTERM); + } + } +} + +fn wait_for_child(pid: i32) -> Result { + let mut status: libc::c_int = 0; + let rc = unsafe { libc::waitpid(pid, &raw mut status, 0) }; + if rc < 0 { + return Err(format!( + "waitpid({pid}) failed: {}", + std::io::Error::last_os_error() + )); + } + Ok(status) +} + +fn cleanup_gvproxy(mut guard: Option) { + if let Some(mut guard) = guard.take() + && let Some(mut child) = guard.disarm() + { + let _ = child.kill(); + let _ = child.wait(); + } +} + +fn check(ret: i32, func: &'static str) -> Result<(), String> { + if ret < 0 { + Err(format!("{func} failed with error code {ret}")) + } else { + Ok(()) + } +} + +fn c_string_array(strings: &[&str]) -> Result<(Vec, Vec<*const libc::c_char>), String> { + let owned: Vec = strings + .iter() + .map(|s| CString::new(*s)) + .collect::, _>>() + .map_err(|e| format!("invalid string array entry: {e}"))?; + let mut ptrs: Vec<*const libc::c_char> = owned.iter().map(|c| c.as_ptr()).collect(); + ptrs.push(ptr::null()); + Ok((owned, ptrs)) +} + +fn path_to_cstring(path: &Path) -> Result { + let path = path + .to_str() + .ok_or_else(|| format!("path is not valid UTF-8: {}", path.display()))?; + CString::new(path).map_err(|e| format!("invalid path string {}: {e}", path)) +} + +#[cfg(target_os = "linux")] +fn check_kvm_access() -> Result<(), String> { + std::fs::OpenOptions::new() + .read(true) + .open("/dev/kvm") + .map(|_| ()) + .map_err(|e| { + format!("cannot open /dev/kvm: {e}\nKVM access is required to run microVMs on Linux.") + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn plan_gvproxy_ports_reuses_sandbox_ssh_mapping() { + let plan = plan_gvproxy_ports(&["64739:2222".to_string()]).expect("plan should succeed"); + + assert_eq!(plan.ssh_port, 64739); + assert!(plan.forwarded_ports.is_empty()); + } + + #[test] + fn plan_gvproxy_ports_keeps_non_ssh_mappings_for_forwarder() { + let plan = plan_gvproxy_ports(&["64739:8080".to_string()]).expect("plan should succeed"); + + assert_ne!(plan.ssh_port, 64739); + assert_eq!(plan.forwarded_ports, vec!["64739:8080".to_string()]); + } + + #[test] + fn plan_gvproxy_ports_ignores_privileged_host_ports_for_direct_ssh() { + let plan = plan_gvproxy_ports(&["22:2222".to_string()]).expect("plan should succeed"); + + assert_ne!(plan.ssh_port, 22); + assert_eq!(plan.forwarded_ports, vec!["22:2222".to_string()]); + } + + #[test] + fn parse_port_mapping_rejects_invalid_entries() { + let err = parse_port_mapping("bad:mapping").expect_err("invalid mapping should fail"); + assert!(err.contains("invalid port mapping")); + } +} diff --git a/crates/openshell-driver-vm/start.sh b/crates/openshell-driver-vm/start.sh new file mode 100755 index 0000000000..aa98874607 --- /dev/null +++ b/crates/openshell-driver-vm/start.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +COMPRESSED_DIR="${ROOT}/target/vm-runtime-compressed" +STATE_DIR_DEFAULT="${ROOT}/target/openshell-vm-driver-dev" +STATE_DIR="${OPENSHELL_VM_DRIVER_STATE_DIR:-${STATE_DIR_DEFAULT}}" +DB_PATH_DEFAULT="${STATE_DIR}/openshell.db" +SERVER_PORT="${OPENSHELL_SERVER_PORT:-8080}" +VM_HOST_GATEWAY_DEFAULT="${OPENSHELL_VM_HOST_GATEWAY:-host.containers.internal}" + +export OPENSHELL_VM_RUNTIME_COMPRESSED_DIR="${OPENSHELL_VM_RUNTIME_COMPRESSED_DIR:-${COMPRESSED_DIR}}" + +mkdir -p "${STATE_DIR}" + +normalize_bool() { + case "${1,,}" in + 1|true|yes|on) echo "true" ;; + 0|false|no|off) echo "false" ;; + *) + echo "invalid boolean value '$1' (expected true/false, 1/0, yes/no, on/off)" >&2 + exit 1 + ;; + esac +} + +if [ ! -f "${COMPRESSED_DIR}/rootfs.tar.zst" ]; then + echo "==> Building base VM rootfs tarball" + mise run vm:rootfs -- --base +fi + +if [ ! -f "${COMPRESSED_DIR}/rootfs.tar.zst" ] || ! find "${COMPRESSED_DIR}" -maxdepth 1 -name 'libkrun*.zst' | grep -q .; then + echo "==> Preparing embedded VM runtime" + mise run vm:setup +fi + +echo "==> Building gateway and VM compute driver" +cargo build -p openshell-server -p openshell-driver-vm + +if [ "$(uname -s)" = "Darwin" ]; then + echo "==> Codesigning VM compute driver" + codesign \ + --entitlements "${ROOT}/crates/openshell-driver-vm/entitlements.plist" \ + --force \ + -s - \ + "${ROOT}/target/debug/openshell-driver-vm" +fi + +export OPENSHELL_DISABLE_TLS="$(normalize_bool "${OPENSHELL_DISABLE_TLS:-true}")" +export OPENSHELL_DB_URL="${OPENSHELL_DB_URL:-sqlite:${DB_PATH_DEFAULT}}" +export OPENSHELL_DRIVERS="${OPENSHELL_DRIVERS:-vm}" +export OPENSHELL_GRPC_ENDPOINT="${OPENSHELL_GRPC_ENDPOINT:-http://${VM_HOST_GATEWAY_DEFAULT}:${SERVER_PORT}}" +export OPENSHELL_SSH_GATEWAY_HOST="${OPENSHELL_SSH_GATEWAY_HOST:-127.0.0.1}" +export OPENSHELL_SSH_GATEWAY_PORT="${OPENSHELL_SSH_GATEWAY_PORT:-${SERVER_PORT}}" +export OPENSHELL_SSH_HANDSHAKE_SECRET="${OPENSHELL_SSH_HANDSHAKE_SECRET:-dev-vm-driver-secret}" +export OPENSHELL_VM_DRIVER_STATE_DIR="${STATE_DIR}" +export OPENSHELL_VM_COMPUTE_DRIVER_BIN="${OPENSHELL_VM_COMPUTE_DRIVER_BIN:-${ROOT}/target/debug/openshell-driver-vm}" + +echo "==> Starting OpenShell server with VM compute driver" +exec "${ROOT}/target/debug/openshell-gateway" diff --git a/crates/openshell-server/Cargo.toml b/crates/openshell-server/Cargo.toml index 33e3542475..29f99d0092 100644 --- a/crates/openshell-server/Cargo.toml +++ b/crates/openshell-server/Cargo.toml @@ -65,6 +65,7 @@ tokio-stream = { workspace = true } sqlx = { workspace = true } reqwest = { workspace = true } uuid = { workspace = true } +url = { workspace = true } hmac = "0.12" sha2 = "0.10" hex = "0.4" diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index 9509fe84b5..ba94250368 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -11,6 +11,7 @@ use std::path::PathBuf; use tracing::info; use tracing_subscriber::EnvFilter; +use crate::compute::VmComputeConfig; use crate::{run_server, tracing_bus::TracingLogBus}; /// `OpenShell` gateway process - gRPC and HTTP server with protocol multiplexing. @@ -112,6 +113,54 @@ struct Args { #[arg(long, env = "OPENSHELL_HOST_GATEWAY_IP")] host_gateway_ip: Option, + /// Working directory for VM driver sandbox state. + #[arg( + long, + env = "OPENSHELL_VM_DRIVER_STATE_DIR", + default_value_os_t = VmComputeConfig::default_state_dir() + )] + vm_driver_state_dir: PathBuf, + + /// VM compute-driver binary spawned by the gateway. + #[arg(long, env = "OPENSHELL_VM_COMPUTE_DRIVER_BIN")] + vm_compute_driver_bin: Option, + + /// libkrun log level used by the VM helper. + #[arg( + long, + env = "OPENSHELL_VM_KRUN_LOG_LEVEL", + default_value_t = VmComputeConfig::default_krun_log_level() + )] + vm_krun_log_level: u32, + + /// Default vCPU count for VM sandboxes. + #[arg( + long, + env = "OPENSHELL_VM_DRIVER_VCPUS", + default_value_t = VmComputeConfig::default_vcpus() + )] + vm_vcpus: u8, + + /// Default memory allocation for VM sandboxes, in MiB. + #[arg( + long, + env = "OPENSHELL_VM_DRIVER_MEM_MIB", + default_value_t = VmComputeConfig::default_mem_mib() + )] + vm_mem_mib: u32, + + /// CA certificate installed into VM sandboxes for gateway mTLS. + #[arg(long, env = "OPENSHELL_VM_TLS_CA")] + vm_tls_ca: Option, + + /// Client certificate installed into VM sandboxes for gateway mTLS. + #[arg(long, env = "OPENSHELL_VM_TLS_CERT")] + vm_tls_cert: Option, + + /// Client private key installed into VM sandboxes for gateway mTLS. + #[arg(long, env = "OPENSHELL_VM_TLS_KEY")] + vm_tls_key: Option, + /// Disable TLS entirely — listen on plaintext HTTP. /// Use this when the gateway sits behind a reverse proxy or tunnel /// (e.g. Cloudflare Tunnel) that terminates TLS at the edge. @@ -211,6 +260,17 @@ async fn run_from_args(args: Args) -> Result<()> { config = config.with_host_gateway_ip(ip); } + let vm_config = VmComputeConfig { + state_dir: args.vm_driver_state_dir, + compute_driver_bin: args.vm_compute_driver_bin, + krun_log_level: args.vm_krun_log_level, + vcpus: args.vm_vcpus, + mem_mib: args.vm_mem_mib, + guest_tls_ca: args.vm_tls_ca, + guest_tls_cert: args.vm_tls_cert, + guest_tls_key: args.vm_tls_key, + }; + if args.disable_tls { info!("TLS disabled — listening on plaintext HTTP"); } else if args.disable_gateway_auth { @@ -219,7 +279,9 @@ async fn run_from_args(args: Args) -> Result<()> { info!(bind = %config.bind_address, "Starting OpenShell server"); - run_server(config, tracing_log_bus).await.into_diagnostic() + run_server(config, vm_config, tracing_log_bus) + .await + .into_diagnostic() } fn parse_compute_driver(value: &str) -> std::result::Result { diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 846782c65e..6b55740160 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -3,6 +3,10 @@ //! Gateway-owned compute orchestration over a pluggable compute backend. +pub mod vm; + +pub use vm::VmComputeConfig; + use crate::grpc::policy::{SANDBOX_SETTINGS_OBJECT_TYPE, sandbox_settings_id}; use crate::persistence::{ObjectId, ObjectName, ObjectRecord, ObjectType, Store}; use crate::sandbox_index::SandboxIndex; @@ -14,8 +18,8 @@ use openshell_core::proto::compute::v1::{ DriverResourceRequirements, DriverSandbox, DriverSandboxSpec, DriverSandboxStatus, DriverSandboxTemplate, GetCapabilitiesRequest, GetSandboxRequest, ListSandboxesRequest, ResolveSandboxEndpointRequest, ResolveSandboxEndpointResponse, ValidateSandboxCreateRequest, - WatchSandboxesEvent, WatchSandboxesRequest, compute_driver_server::ComputeDriver, - sandbox_endpoint, watch_sandboxes_event, + WatchSandboxesEvent, WatchSandboxesRequest, compute_driver_client::ComputeDriverClient, + compute_driver_server::ComputeDriver, sandbox_endpoint, watch_sandboxes_event, }; use openshell_core::proto::{ PlatformEvent, Sandbox, SandboxCondition, SandboxPhase, SandboxSpec, SandboxStatus, @@ -31,6 +35,7 @@ use std::pin::Pin; use std::sync::Arc; use std::time::Duration; use tokio::sync::Mutex; +use tonic::transport::Channel; use tonic::{Code, Request, Status}; use tracing::{info, warn}; @@ -54,16 +59,145 @@ pub enum ComputeError { #[error("{0}")] Message(String), } - #[derive(Debug)] pub enum ResolvedEndpoint { Ip(IpAddr, u16), Host(String, u16), } +#[derive(Debug)] +pub(crate) struct ManagedDriverProcess { + child: std::sync::Mutex>, + socket_path: std::path::PathBuf, +} + +impl ManagedDriverProcess { + pub(crate) fn new(child: tokio::process::Child, socket_path: std::path::PathBuf) -> Self { + Self { + child: std::sync::Mutex::new(Some(child)), + socket_path, + } + } +} + +impl Drop for ManagedDriverProcess { + fn drop(&mut self) { + if let Ok(mut child) = self.child.lock() { + let _ = child.take(); + } + let _ = std::fs::remove_file(&self.socket_path); + } +} + +#[derive(Debug, Clone)] +struct RemoteComputeDriver { + channel: Channel, +} + +impl RemoteComputeDriver { + fn new(channel: Channel) -> Self { + Self { channel } + } + + fn client(&self) -> ComputeDriverClient { + ComputeDriverClient::new(self.channel.clone()) + } +} + +#[tonic::async_trait] +impl ComputeDriver for RemoteComputeDriver { + type WatchSandboxesStream = DriverWatchStream; + + async fn get_capabilities( + &self, + request: Request, + ) -> Result, Status> + { + let mut client = self.client(); + client.get_capabilities(request).await + } + + async fn validate_sandbox_create( + &self, + request: Request, + ) -> Result< + tonic::Response, + Status, + > { + let mut client = self.client(); + client.validate_sandbox_create(request).await + } + + async fn get_sandbox( + &self, + request: Request, + ) -> Result, Status> + { + let mut client = self.client(); + client.get_sandbox(request).await + } + + async fn list_sandboxes( + &self, + request: Request, + ) -> Result, Status> + { + let mut client = self.client(); + client.list_sandboxes(request).await + } + + async fn create_sandbox( + &self, + request: Request, + ) -> Result, Status> + { + let mut client = self.client(); + client.create_sandbox(request).await + } + + async fn stop_sandbox( + &self, + request: Request, + ) -> Result, Status> + { + let mut client = self.client(); + client.stop_sandbox(request).await + } + + async fn delete_sandbox( + &self, + request: Request, + ) -> Result, Status> + { + let mut client = self.client(); + client.delete_sandbox(request).await + } + + async fn resolve_sandbox_endpoint( + &self, + request: Request, + ) -> Result, Status> { + let mut client = self.client(); + client.resolve_sandbox_endpoint(request).await + } + + async fn watch_sandboxes( + &self, + request: Request, + ) -> Result, Status> { + let mut client = self.client(); + let response = client.watch_sandboxes(request).await?; + let stream = response + .into_inner() + .map(|item| item.map_err(|status| status)); + Ok(tonic::Response::new(Box::pin(stream))) + } +} + #[derive(Clone)] pub struct ComputeRuntime { driver: SharedComputeDriver, + _driver_process: Option>, default_image: String, store: Arc, sandbox_index: SandboxIndex, @@ -79,17 +213,14 @@ impl fmt::Debug for ComputeRuntime { } impl ComputeRuntime { - pub async fn new_kubernetes( - config: KubernetesComputeConfig, + async fn from_driver( + driver: SharedComputeDriver, + driver_process: Option>, store: Arc, sandbox_index: SandboxIndex, sandbox_watch_bus: SandboxWatchBus, tracing_log_bus: TracingLogBus, ) -> Result { - let driver = KubernetesComputeDriver::new(config) - .await - .map_err(|err| ComputeError::Message(err.to_string()))?; - let driver: SharedComputeDriver = Arc::new(ComputeDriverService::new(driver)); let default_image = driver .get_capabilities(Request::new(GetCapabilitiesRequest {})) .await @@ -98,6 +229,7 @@ impl ComputeRuntime { .default_image; Ok(Self { driver, + _driver_process: driver_process, default_image, store, sandbox_index, @@ -107,6 +239,48 @@ impl ComputeRuntime { }) } + pub async fn new_kubernetes( + config: KubernetesComputeConfig, + store: Arc, + sandbox_index: SandboxIndex, + sandbox_watch_bus: SandboxWatchBus, + tracing_log_bus: TracingLogBus, + ) -> Result { + let driver = KubernetesComputeDriver::new(config) + .await + .map_err(|err| ComputeError::Message(err.to_string()))?; + let driver: SharedComputeDriver = Arc::new(ComputeDriverService::new(driver)); + Self::from_driver( + driver, + None, + store, + sandbox_index, + sandbox_watch_bus, + tracing_log_bus, + ) + .await + } + + pub(crate) async fn new_remote_vm( + channel: Channel, + driver_process: Option>, + store: Arc, + sandbox_index: SandboxIndex, + sandbox_watch_bus: SandboxWatchBus, + tracing_log_bus: TracingLogBus, + ) -> Result { + let driver: SharedComputeDriver = Arc::new(RemoteComputeDriver::new(channel)); + Self::from_driver( + driver, + driver_process, + store, + sandbox_index, + sandbox_watch_bus, + tracing_log_bus, + ) + .await + } + #[must_use] pub fn default_image(&self) -> &str { &self.default_image @@ -919,7 +1093,7 @@ fn rewrite_user_facing_conditions(status: &mut Option, spec: Opti fn is_terminal_failure_reason(reason: &str) -> bool { let reason = reason.to_ascii_lowercase(); - let transient_reasons = ["reconcilererror", "dependenciesnotready"]; + let transient_reasons = ["reconcilererror", "dependenciesnotready", "starting"]; !transient_reasons.contains(&reason.as_str()) } @@ -1061,6 +1235,7 @@ mod tests { let store = Arc::new(Store::connect("sqlite::memory:").await.unwrap()); ComputeRuntime { driver, + _driver_process: None, default_image: "openshell/sandbox:test".to_string(), store, sandbox_index: SandboxIndex::new(), @@ -1134,6 +1309,7 @@ mod tests { "Pod exists with phase: Pending; Service Exists", ), ("dependenciesnotready", "lowercase also works"), + ("Starting", "VM is starting"), ]; for (reason, message) in transient_cases { @@ -1170,6 +1346,7 @@ mod tests { "DependenciesNotReady", "Pod exists with phase: Pending; Service Exists", ), + ("Starting", "VM is starting"), ]; for (reason, message) in transient_conditions { diff --git a/crates/openshell-server/src/compute/vm.rs b/crates/openshell-server/src/compute/vm.rs new file mode 100644 index 0000000000..d0f397b01e --- /dev/null +++ b/crates/openshell-server/src/compute/vm.rs @@ -0,0 +1,429 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! VM compute driver plumbing. +//! +//! This module owns everything needed to hand the gateway a `Channel` speaking +//! the `openshell.compute.v1.ComputeDriver` RPC surface against an +//! `openshell-driver-vm` subprocess over a Unix domain socket: +//! +//! - [`VmComputeConfig`]: gateway-local configuration (state dir, driver binary, +//! VM shape, guest TLS material). +//! - [`spawn`]: spawn the driver subprocess, wait for its UDS to be ready, +//! and return a live gRPC channel plus a [`ManagedDriverProcess`] handle +//! that will reap the subprocess and clean up the socket on drop. +//! - Helpers to resolve the driver binary, compute the socket path, and +//! validate guest TLS material when the gateway runs an `https://` control +//! plane. +//! +//! The VM-driver fields deliberately live here rather than in +//! [`openshell_core::Config`] so the shared core stays free of driver-specific +//! plumbing. +//! +//! TODO(driver-abstraction): this module still assumes the concrete VM driver +//! (argv shape, guest-TLS flags, libkrun-specific settings). Once we land the +//! generalized compute-driver interface, the CLI-arg plumbing below should +//! be replaced with a driver-agnostic launcher that speaks gRPC to +//! configure the driver — and this file should collapse to the types that +//! are genuinely VM-specific (libkrun log level, vCPU / memory shape) plus a +//! trait implementation registering the VM driver against the generic +//! interface. + +#[cfg(unix)] +use super::ManagedDriverProcess; +#[cfg(unix)] +use hyper_util::rt::TokioIo; +#[cfg(unix)] +use openshell_core::proto::compute::v1::{ + GetCapabilitiesRequest, compute_driver_client::ComputeDriverClient, +}; +use openshell_core::{Config, Error, Result}; +use std::path::PathBuf; +#[cfg(unix)] +use std::{io::ErrorKind, process::Stdio, sync::Arc, time::Duration}; +#[cfg(unix)] +use tokio::net::UnixStream; +#[cfg(unix)] +use tokio::process::Command; +use tonic::transport::Channel; +#[cfg(unix)] +use tonic::transport::Endpoint; +#[cfg(unix)] +use tower::service_fn; + +/// Configuration for launching and talking to the VM compute driver. +#[derive(Debug, Clone)] +pub struct VmComputeConfig { + /// Working directory for VM driver sandbox state. + pub state_dir: PathBuf, + + /// Optional override for the `openshell-driver-vm` binary path. + /// When `None`, the gateway resolves a sibling of its own executable. + pub compute_driver_bin: Option, + + /// libkrun log level used by the VM driver helper. + pub krun_log_level: u32, + + /// Default vCPU count for VM sandboxes. + pub vcpus: u8, + + /// Default memory allocation for VM sandboxes, in MiB. + pub mem_mib: u32, + + /// Host-side CA certificate for the guest's mTLS client bundle. + pub guest_tls_ca: Option, + + /// Host-side client certificate for the guest's mTLS client bundle. + pub guest_tls_cert: Option, + + /// Host-side private key for the guest's mTLS client bundle. + pub guest_tls_key: Option, +} + +impl VmComputeConfig { + /// Default working directory for VM driver state. + #[must_use] + pub fn default_state_dir() -> PathBuf { + PathBuf::from("target/openshell-vm-driver") + } + + /// Default libkrun log level. + #[must_use] + pub const fn default_krun_log_level() -> u32 { + 1 + } + + /// Default vCPU count. + #[must_use] + pub const fn default_vcpus() -> u8 { + 2 + } + + /// Default memory allocation, in MiB. + #[must_use] + pub const fn default_mem_mib() -> u32 { + 2048 + } +} + +impl Default for VmComputeConfig { + fn default() -> Self { + Self { + state_dir: Self::default_state_dir(), + compute_driver_bin: None, + krun_log_level: Self::default_krun_log_level(), + vcpus: Self::default_vcpus(), + mem_mib: Self::default_mem_mib(), + guest_tls_ca: None, + guest_tls_cert: None, + guest_tls_key: None, + } + } +} + +#[cfg(unix)] +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct VmGuestTlsPaths { + pub(crate) ca: PathBuf, + pub(crate) cert: PathBuf, + pub(crate) key: PathBuf, +} + +/// Resolve the `openshell-driver-vm` binary path, falling back to a sibling +/// of the gateway's own executable when an override is not supplied. +pub(crate) fn resolve_compute_driver_bin(vm_config: &VmComputeConfig) -> Result { + let path = if let Some(path) = vm_config.compute_driver_bin.clone() { + path + } else { + let current_exe = std::env::current_exe() + .map_err(|e| Error::config(format!("failed to resolve current executable: {e}")))?; + let Some(parent) = current_exe.parent() else { + return Err(Error::config(format!( + "current executable '{}' has no parent directory", + current_exe.display() + ))); + }; + parent.join("openshell-driver-vm") + }; + + if !path.is_file() { + return Err(Error::config(format!( + "vm compute driver binary '{}' does not exist; set --vm-compute-driver-bin or OPENSHELL_VM_COMPUTE_DRIVER_BIN", + path.display() + ))); + } + + Ok(path) +} + +/// Path of the Unix domain socket the driver will listen on. +pub(crate) fn compute_driver_socket_path(vm_config: &VmComputeConfig) -> PathBuf { + vm_config.state_dir.join("compute-driver.sock") +} + +#[cfg(unix)] +pub(crate) fn compute_driver_guest_tls_paths( + config: &Config, + vm_config: &VmComputeConfig, +) -> Result> { + if !config.grpc_endpoint.starts_with("https://") { + return Ok(None); + } + + let provided = [ + vm_config.guest_tls_ca.as_ref(), + vm_config.guest_tls_cert.as_ref(), + vm_config.guest_tls_key.as_ref(), + ]; + if provided.iter().all(Option::is_none) { + return Err(Error::config( + "vm compute driver requires --vm-tls-ca, --vm-tls-cert, and --vm-tls-key when OPENSHELL_GRPC_ENDPOINT uses https://", + )); + } + + let Some(ca) = vm_config.guest_tls_ca.clone() else { + return Err(Error::config( + "--vm-tls-ca is required when VM guest TLS materials are configured", + )); + }; + let Some(cert) = vm_config.guest_tls_cert.clone() else { + return Err(Error::config( + "--vm-tls-cert is required when VM guest TLS materials are configured", + )); + }; + let Some(key) = vm_config.guest_tls_key.clone() else { + return Err(Error::config( + "--vm-tls-key is required when VM guest TLS materials are configured", + )); + }; + + for path in [&ca, &cert, &key] { + if !path.is_file() { + return Err(Error::config(format!( + "vm guest TLS material '{}' does not exist or is not a file", + path.display() + ))); + } + } + + Ok(Some(VmGuestTlsPaths { ca, cert, key })) +} + +/// Launch the VM compute-driver subprocess, wait for its UDS to come up, +/// and return a gRPC `Channel` connected to it plus a process handle that +/// kills the subprocess and removes the socket on drop. +#[cfg(unix)] +pub(crate) async fn spawn( + config: &Config, + vm_config: &VmComputeConfig, +) -> Result<(Channel, Arc)> { + if config.grpc_endpoint.trim().is_empty() { + return Err(Error::config( + "grpc_endpoint is required when using the vm compute driver", + )); + } + + let driver_bin = resolve_compute_driver_bin(vm_config)?; + let socket_path = compute_driver_socket_path(vm_config); + let guest_tls_paths = compute_driver_guest_tls_paths(config, vm_config)?; + if let Some(parent) = socket_path.parent() { + std::fs::create_dir_all(parent).map_err(|e| { + Error::execution(format!( + "failed to create vm compute driver socket dir '{}': {e}", + parent.display() + )) + })?; + } + match std::fs::remove_file(&socket_path) { + Ok(()) => {} + Err(err) if err.kind() == ErrorKind::NotFound => {} + Err(err) => { + return Err(Error::execution(format!( + "failed to remove stale vm compute driver socket '{}': {err}", + socket_path.display() + ))); + } + } + + let mut command = Command::new(&driver_bin); + command.kill_on_drop(true); + command.stdin(Stdio::null()); + command.stdout(Stdio::inherit()); + command.stderr(Stdio::inherit()); + command.arg("--bind-socket").arg(&socket_path); + command.arg("--log-level").arg(&config.log_level); + command + .arg("--openshell-endpoint") + .arg(&config.grpc_endpoint); + command.arg("--state-dir").arg(&vm_config.state_dir); + command + .arg("--ssh-handshake-secret") + .arg(&config.ssh_handshake_secret); + command + .arg("--ssh-handshake-skew-secs") + .arg(config.ssh_handshake_skew_secs.to_string()); + command + .arg("--krun-log-level") + .arg(vm_config.krun_log_level.to_string()); + command.arg("--vcpus").arg(vm_config.vcpus.to_string()); + command.arg("--mem-mib").arg(vm_config.mem_mib.to_string()); + if let Some(tls) = guest_tls_paths { + command.arg("--guest-tls-ca").arg(tls.ca); + command.arg("--guest-tls-cert").arg(tls.cert); + command.arg("--guest-tls-key").arg(tls.key); + } + + let mut child = command.spawn().map_err(|e| { + Error::execution(format!( + "failed to launch vm compute driver '{}': {e}", + driver_bin.display() + )) + })?; + let channel = wait_for_compute_driver(&socket_path, &mut child).await?; + let process = Arc::new(ManagedDriverProcess::new(child, socket_path)); + Ok((channel, process)) +} + +#[cfg(not(unix))] +pub(crate) async fn spawn( + _config: &Config, + _vm_config: &VmComputeConfig, +) -> Result<(Channel, std::sync::Arc)> { + Err(Error::config( + "the vm compute driver requires unix domain socket support", + )) +} + +#[cfg(unix)] +async fn wait_for_compute_driver( + socket_path: &std::path::Path, + child: &mut tokio::process::Child, +) -> Result { + let mut last_error: Option = None; + for _ in 0..100 { + if let Some(status) = child.try_wait().map_err(|e| { + Error::execution(format!("failed to poll vm compute driver process: {e}")) + })? { + return Err(Error::execution(format!( + "vm compute driver exited before becoming ready with status {status}" + ))); + } + + match connect_compute_driver(socket_path).await { + Ok(channel) => { + let mut client = ComputeDriverClient::new(channel.clone()); + match client + .get_capabilities(tonic::Request::new(GetCapabilitiesRequest {})) + .await + { + Ok(_) => return Ok(channel), + Err(status) => last_error = Some(status.to_string()), + } + } + Err(err) => last_error = Some(err.to_string()), + } + + tokio::time::sleep(Duration::from_millis(100)).await; + } + + Err(Error::execution(format!( + "timed out waiting for vm compute driver socket '{}': {}", + socket_path.display(), + last_error.unwrap_or_else(|| "unknown error".to_string()) + ))) +} + +#[cfg(unix)] +async fn connect_compute_driver(socket_path: &std::path::Path) -> Result { + let socket_path = socket_path.to_path_buf(); + let display_path = socket_path.clone(); + Endpoint::from_static("http://[::]:50051") + .connect_with_connector(service_fn(move |_: tonic::transport::Uri| { + let socket_path = socket_path.clone(); + async move { UnixStream::connect(socket_path).await.map(TokioIo::new) } + })) + .await + .map_err(|e| { + Error::execution(format!( + "failed to connect to vm compute driver socket '{}': {e}", + display_path.display() + )) + }) +} + +#[cfg(all(test, unix))] +mod tests { + use super::{VmComputeConfig, compute_driver_guest_tls_paths}; + use openshell_core::{Config, TlsConfig}; + use tempfile::tempdir; + + #[test] + fn vm_compute_driver_tls_requires_explicit_guest_bundle() { + let dir = tempdir().unwrap(); + let server_cert = dir.path().join("server.crt"); + let server_key = dir.path().join("server.key"); + let server_ca = dir.path().join("client-ca.crt"); + std::fs::write(&server_cert, "server-cert").unwrap(); + std::fs::write(&server_key, "server-key").unwrap(); + std::fs::write(&server_ca, "client-ca").unwrap(); + + let config = Config::new(Some(TlsConfig { + cert_path: server_cert, + key_path: server_key, + client_ca_path: server_ca, + allow_unauthenticated: false, + })) + .with_grpc_endpoint("https://gateway.internal:8443"); + + let err = compute_driver_guest_tls_paths(&config, &VmComputeConfig::default()) + .expect_err("https vm endpoints should require an explicit guest client bundle"); + assert!( + err.to_string() + .contains("--vm-tls-ca, --vm-tls-cert, and --vm-tls-key") + ); + } + + #[test] + fn vm_compute_driver_tls_uses_guest_bundle_not_gateway_server_identity() { + let dir = tempdir().unwrap(); + let server_cert = dir.path().join("server.crt"); + let server_key = dir.path().join("server.key"); + let server_ca = dir.path().join("client-ca.crt"); + let guest_ca = dir.path().join("guest-ca.crt"); + let guest_cert = dir.path().join("guest.crt"); + let guest_key = dir.path().join("guest.key"); + for path in [ + &server_cert, + &server_key, + &server_ca, + &guest_ca, + &guest_cert, + &guest_key, + ] { + std::fs::write(path, path.display().to_string()).unwrap(); + } + + let config = Config::new(Some(TlsConfig { + cert_path: server_cert.clone(), + key_path: server_key.clone(), + client_ca_path: server_ca, + allow_unauthenticated: false, + })) + .with_grpc_endpoint("https://gateway.internal:8443"); + let vm_config = VmComputeConfig { + guest_tls_ca: Some(guest_ca.clone()), + guest_tls_cert: Some(guest_cert.clone()), + guest_tls_key: Some(guest_key.clone()), + ..Default::default() + }; + + let guest_paths = compute_driver_guest_tls_paths(&config, &vm_config) + .unwrap() + .expect("https vm endpoints should pass an explicit guest client bundle"); + assert_eq!(guest_paths.ca, guest_ca); + assert_eq!(guest_paths.cert, guest_cert); + assert_eq!(guest_paths.key, guest_key); + assert_ne!(guest_paths.cert, server_cert); + assert_ne!(guest_paths.key, server_key); + } +} diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index 8e59308265..0a729c099f 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -930,6 +930,15 @@ fn is_safe_ssh_proxy_target(ip: std::net::IpAddr) -> bool { } } +fn is_explicit_loopback_exec_target(host: &str) -> bool { + if host.eq_ignore_ascii_case("localhost") { + return true; + } + + host.parse::() + .is_ok_and(|ip| ip.is_loopback()) +} + async fn start_single_use_ssh_proxy( target_host: &str, target_port: u16, @@ -938,6 +947,7 @@ async fn start_single_use_ssh_proxy( let listener = TcpListener::bind(("127.0.0.1", 0)).await?; let port = listener.local_addr()?.port(); let target_host = target_host.to_string(); + let allow_explicit_loopback = is_explicit_loopback_exec_target(target_host.as_str()); let handshake_secret = handshake_secret.to_string(); let task = tokio::spawn(async move { @@ -962,7 +972,7 @@ async fn start_single_use_ssh_proxy( } }; - if !is_safe_ssh_proxy_target(resolved.ip()) { + if !allow_explicit_loopback && !is_safe_ssh_proxy_target(resolved.ip()) { warn!( target_host = %target_host, resolved_ip = %resolved.ip(), @@ -1214,6 +1224,21 @@ mod tests { assert!(!is_safe_ssh_proxy_target(ip)); } + #[test] + fn explicit_loopback_exec_target_is_allowed() { + assert!(is_explicit_loopback_exec_target("127.0.0.1")); + assert!(is_explicit_loopback_exec_target("::1")); + assert!(is_explicit_loopback_exec_target("localhost")); + } + + #[test] + fn non_loopback_exec_target_is_not_treated_as_explicit_loopback() { + assert!(!is_explicit_loopback_exec_target("10.0.0.5")); + assert!(!is_explicit_loopback_exec_target( + "sandbox.default.svc.cluster.local" + )); + } + // ---- petname / generate_name ---- #[test] diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 7549a17740..da56a5c940 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -8,6 +8,16 @@ //! - HTTP health endpoints //! - Protocol multiplexing (gRPC + HTTP on same port) //! - mTLS support +//! +//! TODO(driver-abstraction): `build_compute_runtime` still switches on +//! [`ComputeDriverKind`] and calls driver-specific constructors +//! ([`ComputeRuntime::new_kubernetes`], [`compute::vm::spawn`] + +//! [`ComputeRuntime::new_remote_vm`]). Once we have a generalized compute +//! driver interface, the per-arm wiring here should collapse to a single +//! driver-agnostic path that asks each registered driver to produce a +//! [`Channel`](tonic::transport::Channel) and hands the rest of the gateway a +//! uniform [`ComputeRuntime`]. The remaining VM plumbing now lives in +//! [`compute::vm`]; keep this file driver-agnostic going forward. mod auth; pub mod cli; @@ -28,10 +38,11 @@ use openshell_core::{ComputeDriverKind, Config, Error, Result}; use std::collections::HashMap; use std::io::ErrorKind; use std::sync::{Arc, Mutex}; +use std::time::Duration; use tokio::net::TcpListener; use tracing::{debug, error, info}; -use compute::ComputeRuntime; +use compute::{ComputeRuntime, VmComputeConfig}; pub use grpc::OpenShellService; pub use http::{health_router, http_router}; pub use multiplex::{MultiplexService, MultiplexedService}; @@ -115,7 +126,11 @@ impl ServerState { /// # Errors /// /// Returns an error if the server fails to start or encounters a fatal error. -pub async fn run_server(config: Config, tracing_log_bus: TracingLogBus) -> Result<()> { +pub async fn run_server( + config: Config, + vm_config: VmComputeConfig, + tracing_log_bus: TracingLogBus, +) -> Result<()> { let database_url = config.database_url.trim(); if database_url.is_empty() { return Err(Error::config("database_url is required")); @@ -132,6 +147,7 @@ pub async fn run_server(config: Config, tracing_log_bus: TracingLogBus) -> Resul let sandbox_watch_bus = SandboxWatchBus::new(); let compute = build_compute_runtime( &config, + &vm_config, store.clone(), sandbox_index.clone(), sandbox_watch_bus.clone(), @@ -148,7 +164,7 @@ pub async fn run_server(config: Config, tracing_log_bus: TracingLogBus) -> Resul )); state.compute.spawn_watchers(); - ssh_tunnel::spawn_session_reaper(store.clone(), std::time::Duration::from_secs(3600)); + ssh_tunnel::spawn_session_reaper(store.clone(), Duration::from_secs(3600)); // Create the multiplexed service let service = MultiplexService::new(state.clone()); @@ -215,6 +231,7 @@ pub async fn run_server(config: Config, tracing_log_bus: TracingLogBus) -> Resul async fn build_compute_runtime( config: &Config, + vm_config: &VmComputeConfig, store: Arc, sandbox_index: SandboxIndex, sandbox_watch_bus: SandboxWatchBus, @@ -244,6 +261,19 @@ async fn build_compute_runtime( ) .await .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}"))), + ComputeDriverKind::Vm => { + let (channel, driver_process) = compute::vm::spawn(config, vm_config).await?; + ComputeRuntime::new_remote_vm( + channel, + Some(driver_process), + store, + sandbox_index, + sandbox_watch_bus, + tracing_log_bus, + ) + .await + .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}"))) + } ComputeDriverKind::Podman => Err(Error::config( "compute driver 'podman' is not implemented yet", )), @@ -255,7 +285,7 @@ fn configured_compute_driver(config: &Config) -> Result { [] => Err(Error::config( "at least one compute driver must be configured", )), - [driver @ ComputeDriverKind::Kubernetes] => Ok(*driver), + [driver @ ComputeDriverKind::Kubernetes] | [driver @ ComputeDriverKind::Vm] => Ok(*driver), [ComputeDriverKind::Podman] => Err(Error::config( "compute driver 'podman' is not implemented yet", )), @@ -332,4 +362,13 @@ mod tests { .contains("compute driver 'podman' is not implemented yet") ); } + + #[test] + fn configured_compute_driver_accepts_vm() { + let config = Config::new(None).with_compute_drivers([ComputeDriverKind::Vm]); + assert_eq!( + configured_compute_driver(&config).unwrap(), + ComputeDriverKind::Vm + ); + } } diff --git a/deploy/docker/Dockerfile.driver-vm-macos b/deploy/docker/Dockerfile.driver-vm-macos new file mode 100644 index 0000000000..ac0aec9523 --- /dev/null +++ b/deploy/docker/Dockerfile.driver-vm-macos @@ -0,0 +1,118 @@ +# syntax=docker/dockerfile:1.6 + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Cross-compile the openshell-driver-vm binary for macOS aarch64 (Apple +# Silicon) using the osxcross toolchain. +# +# openshell-driver-vm loads libkrun/libkrunfw at runtime via dlopen, so it +# does NOT need Hypervisor.framework headers at build time. Pre-compressed +# runtime artifacts (libkrun, libkrunfw, gvproxy, rootfs) are injected via +# the vm-runtime-compressed build context and embedded into the binary via +# include_bytes!(). +# +# Usage: +# docker buildx build -f deploy/docker/Dockerfile.driver-vm-macos \ +# --build-arg OPENSHELL_CARGO_VERSION=0.6.0 \ +# --build-context vm-runtime-compressed=/path/to/compressed-dir \ +# --output type=local,dest=out/ . + +ARG OSXCROSS_IMAGE=crazymax/osxcross:latest + +FROM ${OSXCROSS_IMAGE} AS osxcross + +FROM python:3.12-slim AS builder + +ARG CARGO_TARGET_CACHE_SCOPE=default + +ENV PATH="/root/.cargo/bin:/usr/local/bin:/osxcross/bin:${PATH}" +ENV LD_LIBRARY_PATH="/osxcross/lib" + +COPY --from=osxcross /osxcross /osxcross + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + clang \ + cmake \ + curl \ + pkg-config \ + && rm -rf /var/lib/apt/lists/* + +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.88.0 + +RUN rustup target add aarch64-apple-darwin + +WORKDIR /build + +ENV CC_aarch64_apple_darwin=oa64-clang +ENV CXX_aarch64_apple_darwin=oa64-clang++ +ENV AR_aarch64_apple_darwin=aarch64-apple-darwin25.1-ar +ENV CARGO_TARGET_AARCH64_APPLE_DARWIN_LINKER=oa64-clang +ENV CARGO_TARGET_AARCH64_APPLE_DARWIN_AR=aarch64-apple-darwin25.1-ar + +# aws-lc-sys workaround (in case it ends up in the dep tree via feature unification) +RUN ln -sf /osxcross/bin/arm64-apple-darwin25.1-ld /usr/local/bin/arm64-apple-macosx-ld + +# --------------------------------------------------------------------------- +# Stage 1: dependency caching — copy only manifests, create dummy sources, +# build dependencies. This layer is cached unless Cargo.toml/lock changes. +# --------------------------------------------------------------------------- +COPY Cargo.toml Cargo.lock ./ +COPY crates/openshell-driver-vm/Cargo.toml crates/openshell-driver-vm/Cargo.toml +COPY crates/openshell-driver-vm/build.rs crates/openshell-driver-vm/build.rs +COPY crates/openshell-core/Cargo.toml crates/openshell-core/Cargo.toml +COPY crates/openshell-core/build.rs crates/openshell-core/build.rs +COPY proto/ proto/ + +# Scope workspace to the driver + its only internal dep. +RUN sed -i 's|members = \["crates/\*"\]|members = ["crates/openshell-driver-vm", "crates/openshell-core"]|' Cargo.toml + +RUN mkdir -p crates/openshell-driver-vm/src \ + crates/openshell-core/src && \ + echo "fn main() {}" > crates/openshell-driver-vm/src/main.rs && \ + touch crates/openshell-driver-vm/src/lib.rs && \ + touch crates/openshell-core/src/lib.rs + +# Build deps only (cached layer). The 2>/dev/null || true is a warm-cache +# technique; real source is copied in stage 2. +RUN --mount=type=cache,id=cargo-registry-driver-vm-macos,sharing=locked,target=/root/.cargo/registry \ + --mount=type=cache,id=cargo-git-driver-vm-macos,sharing=locked,target=/root/.cargo/git \ + --mount=type=cache,id=cargo-target-driver-vm-macos-${CARGO_TARGET_CACHE_SCOPE},sharing=locked,target=/build/target \ + cargo build --release --target aarch64-apple-darwin -p openshell-driver-vm 2>/dev/null || true + +# --------------------------------------------------------------------------- +# Stage 2: real build with compressed runtime artifacts +# --------------------------------------------------------------------------- +COPY crates/ crates/ + +# Copy compressed VM runtime artifacts for embedding. +# These are passed in via --build-context vm-runtime-compressed=... +COPY --from=vm-runtime-compressed / /build/vm-runtime-compressed/ + +# Touch source files to ensure they're rebuilt (not the cached dummy). +RUN touch crates/openshell-driver-vm/src/main.rs \ + crates/openshell-driver-vm/src/lib.rs \ + crates/openshell-driver-vm/build.rs \ + crates/openshell-core/src/lib.rs \ + crates/openshell-core/build.rs \ + proto/*.proto + +# Declare version ARGs here (not earlier) so the git-hash-bearing values do not +# invalidate the expensive dependency-build layers above on every commit. +ARG OPENSHELL_CARGO_VERSION +ARG OPENSHELL_IMAGE_TAG +RUN --mount=type=cache,id=cargo-registry-driver-vm-macos,sharing=locked,target=/root/.cargo/registry \ + --mount=type=cache,id=cargo-git-driver-vm-macos,sharing=locked,target=/root/.cargo/git \ + --mount=type=cache,id=cargo-target-driver-vm-macos-${CARGO_TARGET_CACHE_SCOPE},sharing=locked,target=/build/target \ + if [ -n "${OPENSHELL_CARGO_VERSION:-}" ]; then \ + sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${OPENSHELL_CARGO_VERSION}"'"/}' Cargo.toml; \ + fi && \ + OPENSHELL_VM_RUNTIME_COMPRESSED_DIR=/build/vm-runtime-compressed \ + OPENSHELL_IMAGE_TAG="${OPENSHELL_IMAGE_TAG:-dev}" \ + cargo build --release --target aarch64-apple-darwin -p openshell-driver-vm && \ + cp target/aarch64-apple-darwin/release/openshell-driver-vm /openshell-driver-vm + +FROM scratch AS binary +COPY --from=builder /openshell-driver-vm /openshell-driver-vm From 2c9c146cb6d522c6c5e415f36def666a0544feeb Mon Sep 17 00:00:00 2001 From: Miyoung Choi Date: Fri, 17 Apr 2026 13:15:29 -0700 Subject: [PATCH 008/142] docs: fix TOC structure (#797) * docs: fix TOC structure * docs: fix wrong section title * remove Home * fix(fern): properly add the landing page --- .../tutorials/first-network-policy.mdx | 2 +- docs/{ => get-started}/tutorials/github-sandbox.mdx | 2 +- docs/{ => get-started}/tutorials/index.mdx | 10 +++++----- .../{ => get-started}/tutorials/inference-ollama.mdx | 2 +- .../tutorials/local-inference-lmstudio.mdx | 2 +- docs/index.mdx | 1 - docs/index.yml | 12 +++++++----- fern/docs.yml | 9 +++++++++ fern/fern.config.json | 2 +- 9 files changed, 26 insertions(+), 16 deletions(-) rename docs/{ => get-started}/tutorials/first-network-policy.mdx (99%) rename docs/{ => get-started}/tutorials/github-sandbox.mdx (99%) rename docs/{ => get-started}/tutorials/index.mdx (73%) rename docs/{ => get-started}/tutorials/inference-ollama.mdx (99%) rename docs/{ => get-started}/tutorials/local-inference-lmstudio.mdx (99%) diff --git a/docs/tutorials/first-network-policy.mdx b/docs/get-started/tutorials/first-network-policy.mdx similarity index 99% rename from docs/tutorials/first-network-policy.mdx rename to docs/get-started/tutorials/first-network-policy.mdx index bb68e6135b..d3d5a416e8 100644 --- a/docs/tutorials/first-network-policy.mdx +++ b/docs/get-started/tutorials/first-network-policy.mdx @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 title: "Write Your First Sandbox Network Policy" sidebar-title: "First Network Policy" -slug: "tutorials/first-network-policy" +slug: "get-started/tutorials/first-network-policy" description: "See how OpenShell network policies work by creating a sandbox, observing default-deny in action, and applying a fine-grained L7 read-only rule." keywords: "Generative AI, Cybersecurity, Tutorial, Policy, Network Policy, Sandbox, Security" --- diff --git a/docs/tutorials/github-sandbox.mdx b/docs/get-started/tutorials/github-sandbox.mdx similarity index 99% rename from docs/tutorials/github-sandbox.mdx rename to docs/get-started/tutorials/github-sandbox.mdx index 0490b6bc1f..8492dde715 100644 --- a/docs/tutorials/github-sandbox.mdx +++ b/docs/get-started/tutorials/github-sandbox.mdx @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 title: "Grant GitHub Push Access to a Sandboxed Agent" sidebar-title: "GitHub Push Access" -slug: "tutorials/github-sandbox" +slug: "get-started/tutorials/github-sandbox" description: "Learn the iterative policy workflow by launching a sandbox, diagnosing a GitHub access denial, and applying a custom policy to fix it." keywords: "Generative AI, Cybersecurity, Tutorial, GitHub, Sandbox, Policy, Claude Code" --- diff --git a/docs/tutorials/index.mdx b/docs/get-started/tutorials/index.mdx similarity index 73% rename from docs/tutorials/index.mdx rename to docs/get-started/tutorials/index.mdx index 970633a887..f5a79b543b 100644 --- a/docs/tutorials/index.mdx +++ b/docs/get-started/tutorials/index.mdx @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 title: "Tutorials" -slug: "tutorials" +slug: "get-started/tutorials" description: "Step-by-step walkthroughs for OpenShell, from first sandbox to production-ready policies." keywords: "Generative AI, Cybersecurity, Tutorial, Sandbox, Policy" position: 1 @@ -12,22 +12,22 @@ Hands-on walkthroughs that teach OpenShell concepts by building real configurati - + Create a sandbox, observe default-deny networking, apply a read-only L7 policy, and inspect audit logs. No AI agent required. - + Launch Claude Code in a sandbox, diagnose a policy denial, and iterate on a custom GitHub policy from outside the sandbox. - + Route inference through Ollama using cloud-hosted or local models, and verify it from a sandbox. - + Route inference to a local LM Studio server via the OpenAI or Anthropic compatible APIs. diff --git a/docs/tutorials/inference-ollama.mdx b/docs/get-started/tutorials/inference-ollama.mdx similarity index 99% rename from docs/tutorials/inference-ollama.mdx rename to docs/get-started/tutorials/inference-ollama.mdx index 5140038fc7..a123a2b28a 100644 --- a/docs/tutorials/inference-ollama.mdx +++ b/docs/get-started/tutorials/inference-ollama.mdx @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 title: "Run Local Inference with Ollama" sidebar-title: "Inference with Ollama" -slug: "tutorials/inference-ollama" +slug: "get-started/tutorials/inference-ollama" description: "Run local and cloud models inside an OpenShell sandbox using the Ollama community sandbox, or route sandbox requests to a host-level Ollama server." keywords: "Generative AI, Cybersecurity, Tutorial, Inference Routing, Ollama, Local Inference, Sandbox" --- diff --git a/docs/tutorials/local-inference-lmstudio.mdx b/docs/get-started/tutorials/local-inference-lmstudio.mdx similarity index 99% rename from docs/tutorials/local-inference-lmstudio.mdx rename to docs/get-started/tutorials/local-inference-lmstudio.mdx index fec37fe2db..7b87df2e5e 100644 --- a/docs/tutorials/local-inference-lmstudio.mdx +++ b/docs/get-started/tutorials/local-inference-lmstudio.mdx @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 title: "Route Local Inference Requests to LM Studio" sidebar-title: "Local Inference with LM Studio" -slug: "tutorials/local-inference-lmstudio" +slug: "get-started/tutorials/local-inference-lmstudio" description: "Configure inference.local to route sandbox requests to a local LM Studio server running on the gateway host." keywords: "Generative AI, Cybersecurity, Tutorial, Inference Routing, LM Studio, Local Inference, Sandbox" --- diff --git a/docs/index.mdx b/docs/index.mdx index 8c09fffb3e..05011e5557 100644 --- a/docs/index.mdx +++ b/docs/index.mdx @@ -2,7 +2,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 title: "NVIDIA OpenShell Developer Guide" -slug: "get-started" description: "OpenShell is the safe, private runtime for autonomous AI agents. Run agents in sandboxed environments that protect your data, credentials, and infrastructure." keywords: "Generative AI, Cybersecurity, AI Agents, Sandboxing, Security, Privacy, Inference Routing" position: 1 diff --git a/docs/index.yml b/docs/index.yml index fd29fc0d84..6453a78398 100644 --- a/docs/index.yml +++ b/docs/index.yml @@ -1,18 +1,20 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +landing-page: + page: "Home" + path: index.mdx + navigation: +- folder: about + title: "About NVIDIA OpenShell" - section: "Get Started" slug: get-started contents: - - page: "Home" - path: index.mdx - page: "Quickstart" path: get-started/quickstart.mdx - - folder: tutorials + - folder: get-started/tutorials skip-slug: true -- folder: about - title: "About NVIDIA OpenShell" - folder: sandboxes - folder: inference title: "Inference Routing" diff --git a/fern/docs.yml b/fern/docs.yml index 010c516791..0aa48c9f76 100644 --- a/fern/docs.yml +++ b/fern/docs.yml @@ -84,3 +84,12 @@ redirects: destination: "/openshell/:path*" - source: "/openshell/:path*.html" destination: "/openshell/:path*" + # tutorials moved under get-started + - source: "/openshell/tutorials" + destination: "/openshell/latest/get-started/tutorials" + - source: "/openshell/tutorials/:path*" + destination: "/openshell/latest/get-started/tutorials/:path*" + - source: "/openshell/latest/tutorials" + destination: "/openshell/latest/get-started/tutorials" + - source: "/openshell/latest/tutorials/:path*" + destination: "/openshell/latest/get-started/tutorials/:path*" diff --git a/fern/fern.config.json b/fern/fern.config.json index 3d47d15e31..d0bf7eb005 100644 --- a/fern/fern.config.json +++ b/fern/fern.config.json @@ -1,4 +1,4 @@ { "organization": "nvidia", - "version": "4.62.4" + "version": "4.75.0" } From b7c763204d8bd9a85be3100939b294729e9bdb04 Mon Sep 17 00:00:00 2001 From: Miyoung Choi Date: Fri, 17 Apr 2026 13:15:59 -0700 Subject: [PATCH 009/142] docs: refresh user-facing docs for recent sandbox and inference changes (#868) * docs: refresh user-facing docs for recent sandbox and inference changes - architecture: document system CA loading for upstream TLS, `tls: skip` as the opt-out, gateway state persistence across restarts, and OCSF structured logging surface. - inference: document per-provider header allowlist, Authorization stripping, 120s streaming idle tolerance, and extended-thinking timeout guidance. - manage-sandboxes: add "Execute a Command in a Sandbox" section for `openshell sandbox exec` with flag reference. - security best practices: expand seccomp denylist (unconditional and conditional blocks), document two-phase Landlock probe, High-severity `landlock-unavailable` finding, and inference keep-alive closure. - observability logging: document port in HTTP log URLs, `[reason:...]` denial suffixes, proxy 403/502 JSON error bodies, and Landlock CONFIG:ENABLED/CONFIG:OTHER events. Signed-off-by: Miyoung Choi * docs(architecture): tighten high-level architecture page Trim implementation detail (CA bundle paths, deprecated TLS keys, SSH handshake secret) from the high-level architecture page, fix accuracy issues surfaced during deep audit, and expand uncommon acronyms on first mention. - Drop unsupported "cost-based routing" claim from Privacy Router row. - Replace "brokers requests across the platform" with auth-boundary description. - Add "inference" to Policy Engine constraint list per AGENTS.md. - Expand Deny rule to include SSRF, blocked control-plane port, and L7 deny paths in addition to deny-by-default. - Switch Allow/Deny labels from hyphen to colon; remove em dashes and a double space. - Expand LLM, SSRF, L7, TLS, CA, PEM, SSH, OCSF, and JSONL on first use. Signed-off-by: Miyoung Choi Made-with: Cursor * docs: address audit feedback on refresh PR - observability/logging: rewrite the allowed_ips paragraph after the Denial Reasons table; the previous wording said authors "can use" invalid entries while also stating they were rejected, which was contradictory and conflated load-time validation with the runtime per-CONNECT denial phrases the section documents. - about/architecture: split compound sentences in the new Gateway Lifecycle and Observability sections so each clause stands alone. - inference/about: drop the streaming-tolerance sentence from the prose paragraph since the dedicated Streaming reliability table row already covers it. Signed-off-by: Miyoung Choi Made-with: Cursor * docs(architecture): address review feedback on architecture page - Drop the Privacy Router row from the Components table; it is not yet a separately exposed customer-facing component. - Update the page description and intro count to match the remaining three components (gateway, sandbox, policy engine). - Split the policy decision into the three modes that the engine actually implements: Explicit Deny (deny rules and hardening rules, takes precedence), Allow, and Implicit Deny (no rule matched). Signed-off-by: Miyoung Choi Made-with: Cursor * docs(architecture): convert policy decisions to a table Promote the three policy decisions (Explicit Deny, Allow, Implicit Deny) to a top-level table with Decision, When it applies, and Outcome columns instead of a nested bulleted list under list item 5. Top-level tables render reliably across markdown renderers, where nested-in-list tables do not. Signed-off-by: Miyoung Choi Made-with: Cursor * docs: repharse a bit --------- Signed-off-by: Miyoung Choi --- docs/about/architecture.mdx | 33 +++++++++----- docs/inference/about.mdx | 5 ++- docs/inference/configure.mdx | 4 +- docs/observability/logging.mdx | 67 +++++++++++++++++++++++++++++ docs/sandboxes/manage-sandboxes.mdx | 32 ++++++++++++++ docs/security/best-practices.mdx | 15 ++++--- 6 files changed, 136 insertions(+), 20 deletions(-) diff --git a/docs/about/architecture.mdx b/docs/about/architecture.mdx index 0c701e6a96..a88e6f490f 100644 --- a/docs/about/architecture.mdx +++ b/docs/about/architecture.mdx @@ -3,25 +3,24 @@ # SPDX-License-Identifier: Apache-2.0 title: "How OpenShell Works" sidebar-title: "How It Works" -description: "OpenShell architecture overview covering the gateway, sandbox, policy engine, and privacy router." +description: "OpenShell architecture overview covering the gateway, sandbox, and policy engine." keywords: "Generative AI, Cybersecurity, AI Agents, Sandboxing, Security, Architecture" position: 2 --- -OpenShell runs inside a Docker container. Each sandbox is an isolated environment managed through the gateway. Four components work together to keep agents secure. +OpenShell runs inside a Docker container. Each sandbox is an isolated environment managed through the gateway. Three components work together to keep agents secure. ![OpenShell architecture diagram](/assets/images/architecture.svg) ## Components -The following table describes each component and its role in the system: +The following table describes each component and its role in the system: | Component | Role | |---|---| -| **Gateway** | Control-plane API that coordinates sandbox lifecycle and state, acts as the auth boundary, and brokers requests across the platform. | +| **Gateway** | Control-plane API that coordinates sandbox lifecycle and state, and acts as the auth boundary between clients and sandboxes. | | **Sandbox** | Isolated runtime that includes container supervision and policy-enforced egress routing. | -| **Policy Engine** | Policy definition and enforcement layer for filesystem, network, and process constraints. Defense in depth enforces policies from the application layer down to infrastructure and kernel layers. | -| **Privacy Router** | Privacy-aware LLM routing layer that keeps sensitive context on sandbox compute and routes based on cost and privacy policy. | +| **Policy Engine** | Policy definition and enforcement layer for filesystem, network, process, and inference constraints. Defense in depth enforces policies from the application layer down to infrastructure and kernel layers. | ## How a Request Flows @@ -31,15 +30,27 @@ Every outbound connection from agent code passes through the same decision path: 2. The proxy inside the sandbox intercepts the connection and identifies which binary opened it. 3. If the target is `https://inference.local`, the proxy handles it as managed inference before policy evaluation. OpenShell strips sandbox-supplied credentials, injects the configured backend credentials, and forwards the request to the managed model endpoint. 4. For every other destination, the proxy queries the policy engine with the destination, port, and calling binary. -5. The policy engine returns one of two decisions: - - **Allow** - the destination and binary match a policy block. Traffic flows directly to the external service. - - **Deny** - no policy block matched. The connection is blocked and logged. +5. The policy engine returns one of the following decisions. Explicit deny takes precedence over allow, and allow takes precedence over implicit deny. -For REST endpoints with TLS termination enabled, the proxy also decrypts TLS and checks each HTTP request against per-method, per-path rules before allowing it through. + | Decision | When it applies | Outcome | + |---|---|---| + | **Explicit Deny** | A deny rule, or a hardening rule such as server-side request forgery (SSRF) protection, a blocked control-plane port, or a Layer 7 (L7) deny, rejects the request even when an allow rule would otherwise permit it. | The connection is blocked and logged. | + | **Allow** | The destination and binary match a policy block, and no deny rule applies. | Traffic flows directly to the external service. | + | **Implicit Deny** | No policy block matched. | The connection is blocked and logged. | + +For REST endpoints, the proxy auto-detects Transport Layer Security (TLS) by peeking the first bytes of each tunnel. When TLS is detected, the proxy terminates it transparently using a per-sandbox ephemeral certificate authority (CA) so each HTTP request can be checked against per-method, per-path rules before forwarding. To trust a corporate or internal CA for upstream traffic, add the PEM-encoded certificate to the sandbox container's trust store at image build time. + +## Gateway Lifecycle + +OpenShell preserves sandbox state across gateway restarts. After `openshell gateway stop` and `openshell gateway start`, running sandboxes resume from their saved state. Existing SSH sessions reconnect without re-authenticating. + +## Observability + +The sandbox emits operational decisions as Open Cybersecurity Schema Framework (OCSF) v1.7.0 structured logs. The events cover network connections, HTTP requests, SSH sessions, process lifecycle, detection findings, and configuration changes. For event classes and the JSON Lines (JSONL) export format, refer to [OCSF JSON Export](/observability/ocsf-json-export). ## Deployment Modes -OpenShell can run locally, on a remote host, or behind a cloud proxy. The architecture is identical in all cases — only the Docker container location and authentication mode change. +OpenShell can run locally, on a remote host, or behind a cloud proxy. The architecture is identical in all cases. Only the Docker container location and authentication mode change. | Mode | Description | Command | |---|---|---| diff --git a/docs/inference/about.mdx b/docs/inference/about.mdx index 7bdc518cc8..97f951e9b1 100644 --- a/docs/inference/about.mdx +++ b/docs/inference/about.mdx @@ -24,10 +24,11 @@ If code calls an external inference host directly, that traffic is evaluated onl | Property | Detail | |---|---| -| Credentials | No sandbox API keys needed. Credentials come from the configured provider record. | -| Header forwarding | `inference.local` forwards only approved inference headers. It strips caller auth and other unexpected headers before sending the request upstream. | +| Credentials | No sandbox API keys needed. Credentials come from the configured provider record. The router strips caller-supplied `Authorization` before forwarding the request. | +| Header forwarding | `inference.local` forwards only a per-provider header allowlist. OpenAI routes allow `openai-organization` and `x-model-id`. Anthropic routes allow `anthropic-version` and `anthropic-beta`. NVIDIA routes allow `x-model-id`. All other caller headers are stripped. | | Configuration | One provider and one model define sandbox inference for the active gateway. Every sandbox on that gateway sees the same `inference.local` backend. | | Provider support | NVIDIA, any OpenAI-compatible provider, and Anthropic all work through the same endpoint. | +| Streaming reliability | The router tolerates idle gaps of up to 120 seconds between streamed chunks so long reasoning responses are not cut off mid-stream. | | Hot-refresh | OpenShell picks up provider credential changes and inference updates without recreating sandboxes. Changes propagate within about 5 seconds by default. | ## Supported API Patterns diff --git a/docs/inference/configure.mdx b/docs/inference/configure.mdx index a90b377dbe..1cc7716dcb 100644 --- a/docs/inference/configure.mdx +++ b/docs/inference/configure.mdx @@ -103,7 +103,7 @@ openshell inference set \ --timeout 300 ``` -The value is in seconds. When `--timeout` is omitted (or set to `0`), the default of 60 seconds applies. +The value is in seconds. When `--timeout` is omitted (or set to `0`), the default of 60 seconds applies. Increase `--timeout` when you expect extended thinking phases so the full response completes before the request deadline. ## Verify the Active Config @@ -154,7 +154,7 @@ response = client.chat.completions.create( ) ``` -The client-supplied `model` and `api_key` values are not sent upstream. The privacy router injects the real credentials from the configured provider and rewrites the model before forwarding. +The client-supplied `model` and `api_key` values are not sent upstream. The privacy router injects the real credentials from the configured provider and rewrites the model before forwarding. Caller-supplied `Authorization` and other non-allowlisted headers are stripped; only a per-provider header allowlist reaches the upstream endpoint. For the exact allowlist, refer to [About Inference Routing](/inference/about). Some SDKs require a non-empty API key even though `inference.local` does not use the sandbox-provided value. In those cases, pass any placeholder such as `test` or `unused`. diff --git a/docs/observability/logging.mdx b/docs/observability/logging.mdx index 25adde3608..2053ba4b2b 100644 --- a/docs/observability/logging.mdx +++ b/docs/observability/logging.mdx @@ -124,6 +124,12 @@ A connection denied because the destination resolves to an always-blocked addres OCSF NET:OPEN [MED] DENIED /usr/bin/curl(1618) -> 169.254.169.254:80 [policy:- engine:ssrf] [reason:resolves to always-blocked address] ``` +An HTTP request to a non-default port. HTTP log URLs include the port whenever it differs from the scheme default (80 for `http`, 443 for `https`): + +```text +OCSF HTTP:GET [INFO] ALLOWED GET http://api.internal.corp:8080/v1/status [policy:internal_api engine:opa] +``` + Proxy and SSH servers ready: ```text @@ -150,6 +156,67 @@ OCSF CONFIG:DETECTED [INFO] Settings poll: config change detected [old_revision: OCSF CONFIG:LOADED [INFO] Policy reloaded successfully [policy_hash:0cc0c2b525573c07] ``` +## Denial Reasons + +Denied `NET:` and `HTTP:` events carry a `[reason:...]` suffix that surfaces the decision detail from the event's `status_detail` field. The reason helps distinguish between policy misses, SSRF hardening, and L7 enforcement without inspecting the full OCSF JSONL record. + +Common reason phrases emitted by the sandbox include: + +| Reason | Meaning | +|---|---| +| `no matching policy` | OPA evaluated the request and no allow rule matched. | +| `resolves to always-blocked address` | The destination resolved to loopback, link-local, or unspecified. These ranges are always blocked, even when listed in `allowed_ips`. | +| `resolves to which is not in allowed_ips, connection rejected` | The destination resolved to an IP outside the policy's `allowed_ips` allowlist. | +| `DNS resolution failed for :` | The proxy could not resolve the destination. | +| `port is a blocked control-plane port, connection rejected` | The destination port matches a control-plane port (etcd, Kubernetes API, kubelet) and is always blocked. | +| `l7 deny` | An L7 policy rule denied the request. | + +Invalid `allowed_ips` entries and entries that overlap always-blocked ranges are rejected at policy-load time, so they never reach the runtime denial path. The phrases above come from the proxy's per-CONNECT `allowed_ips` and SSRF checks, not from policy validation. + +## Proxy Error Responses + +When the HTTP CONNECT proxy denies a request or cannot reach the upstream, it returns an HTTP error response with a JSON body. Clients can parse the body to surface actionable failure details instead of treating the status code alone. + +A denied CONNECT returns `403 Forbidden`: + +```json +{ + "error": "policy_denied", + "detail": "CONNECT api.example.com:443 not permitted by policy" +} +``` + +An upstream that the proxy cannot reach returns `502 Bad Gateway`: + +```json +{ + "error": "upstream_unreachable", + "detail": "connection to api.example.com:443 failed" +} +``` + +The `error` field is a short machine-readable code (`policy_denied`, `ssrf_denied`, `upstream_unreachable`). The `detail` field is a human-readable explanation suitable for display in an agent transcript. + +## Filesystem Sandbox Logs + +Landlock filesystem restrictions emit `CONFIG:` events at startup and whenever the sandbox has to skip a requested path. + +On startup, the probe reports the kernel's supported Landlock ABI version alongside the requested path counts: + +```text +OCSF CONFIG:ENABLED [INFO] Landlock filesystem sandbox available [abi:v2 compat:BestEffort ro:4 rw:2] +OCSF CONFIG:ENABLED [INFO] Applying Landlock filesystem sandbox [abi:V2 compat:BestEffort ro:4 rw:2] +OCSF CONFIG:ENABLED [INFO] Landlock ruleset built [rules_applied:5 skipped:1] +``` + +When `landlock.compatibility` is `best_effort` and a requested path fails to open for reasons other than `NotFound` (for example, permission denied or a symlink loop), the sandbox continues without that path and emits a `[MED]` event so the degradation is not silent: + +```text +OCSF CONFIG:OTHER [MED] Skipping inaccessible Landlock path (best-effort) [path:/opt/data error:Permission denied (os error 13)] +``` + +Set `landlock.compatibility` to `hard_requirement` in the policy to make these failures fatal instead of degraded. + ## Log File Location Inside the sandbox, logs are written to `/var/log/`: diff --git a/docs/sandboxes/manage-sandboxes.mdx b/docs/sandboxes/manage-sandboxes.mdx index b43864401a..1be05e81e3 100644 --- a/docs/sandboxes/manage-sandboxes.mdx +++ b/docs/sandboxes/manage-sandboxes.mdx @@ -70,6 +70,38 @@ When `--editor` is used, OpenShell keeps the sandbox alive and installs an OpenShell-managed SSH include file instead of cluttering your main `~/.ssh/config` with generated host blocks. +## Execute a Command in a Sandbox + +Run a one-shot command inside a running sandbox without opening an interactive shell: + +```shell +openshell sandbox exec -n my-sandbox -- ls -la /workspace +``` + +Pipe stdin into the command: + +```shell +echo "hello" | openshell sandbox exec -n my-sandbox -- cat +``` + +The command's exit code is propagated to the CLI, so `exec` works in scripts that check return codes. + +Run an interactive shell with a TTY: + +```shell +openshell sandbox exec -n my-sandbox --tty -- /bin/bash +``` + +OpenShell allocates a TTY automatically when both stdin and stdout are terminals. Force the behavior with `--tty` or disable it with `--no-tty`. + +| Flag | Purpose | +|---|---| +| `-n`, `--name` | Sandbox to target. | +| `--workdir` | Working directory for the command inside the sandbox. | +| `--timeout` | Command timeout in seconds. `0` disables the timeout. | +| `--tty` | Force TTY allocation. | +| `--no-tty` | Disable TTY allocation even when attached to a terminal. | + ## Monitor and Debug List all sandboxes: diff --git a/docs/security/best-practices.mdx b/docs/security/best-practices.mdx index a84800e4ce..52fc82131c 100644 --- a/docs/security/best-practices.mdx +++ b/docs/security/best-practices.mdx @@ -145,12 +145,14 @@ Paths listed in `read_only` receive read-only access. Paths listed in `read_write` receive full access. All other paths are inaccessible. +Landlock setup runs in two phases. The parent supervisor probes the kernel ABI and opens the configured path file descriptors before forking. The child then applies the ruleset with `restrict_self()` after privilege drop. At startup, OpenShell emits the selected ABI version and the applied read-only and read-write rule counts so you can confirm what the kernel accepted. + | Aspect | Detail | |---|---| -| Default | `compatibility: best_effort`. Uses the highest kernel ABI available. The system skips missing paths with a warning. If the kernel does not support Landlock, the sandbox continues without filesystem restrictions. | +| Default | `compatibility: best_effort`. Uses the highest kernel ABI available. Missing paths are skipped. If the kernel does not support Landlock or any configured path cannot be opened, the sandbox continues without those restrictions and emits a High-severity OCSF `DetectionFinding`. | | What you can change | Set `compatibility: hard_requirement` to abort sandbox startup if Landlock is unavailable or any configured path cannot be opened. | | Risk if relaxed | On kernels without Landlock (pre-5.13), or when all paths fail to open, the sandbox runs without kernel-level filesystem restrictions. The agent can access any file the process user can access. | -| Recommendation | Use `best_effort` for development. Use `hard_requirement` in environments where any gap in filesystem isolation is unacceptable. Run on Ubuntu 22.04+ or any kernel 5.13+ for Landlock support. | +| Recommendation | Use `best_effort` for development. Use `hard_requirement` in environments where any gap in filesystem isolation is unacceptable. Treat High-severity Landlock findings as a signal to investigate the host kernel or the image. Run on Ubuntu 22.04+ or any kernel 5.13+ for Landlock support. | ### Read-Only vs Read-Write Paths @@ -192,13 +194,15 @@ The sandbox process runs as a non-root user after explicit privilege dropping. ### Seccomp Filters -A BPF seccomp filter restricts which socket domains the sandbox process can use. +A BPF seccomp filter restricts which socket domains the sandbox process can use and blocks a denylist of syscalls that enable container escape, privilege escalation, or host observation. The sandbox sets `PR_SET_NO_NEW_PRIVS` before applying the filter. | Aspect | Detail | |---|---| -| Default | The filter allows `AF_INET` and `AF_INET6` (for proxy communication) and blocks `AF_NETLINK`, `AF_PACKET`, `AF_BLUETOOTH`, and `AF_VSOCK` with `EPERM`. The sandbox sets `PR_SET_NO_NEW_PRIVS` before applying the filter. | +| Socket domains | The filter allows `AF_INET` and `AF_INET6` (for proxy communication) and blocks `AF_NETLINK`, `AF_PACKET`, `AF_BLUETOOTH`, and `AF_VSOCK` with `EPERM`. | +| Unconditional syscall blocks | `memfd_create`, `ptrace`, `bpf`, `process_vm_readv`, `process_vm_writev`, `pidfd_open`, `pidfd_getfd`, `pidfd_send_signal`, `io_uring_setup`, `mount`, `fsopen`, `fsconfig`, `fsmount`, `fspick`, `move_mount`, `open_tree`, `setns`, `umount2`, `pivot_root`, `userfaultfd`, `perf_event_open`. | +| Conditional syscall blocks | `execveat` with `AT_EMPTY_PATH`, `unshare` and `clone` with `CLONE_NEWUSER`, and `seccomp(SECCOMP_SET_MODE_FILTER)` are denied with `EPERM`. | | What you can change | This is not a user-facing knob. OpenShell enforces it automatically. | -| Risk if relaxed | `AF_NETLINK` allows manipulation of routing tables and firewall rules. `AF_PACKET` enables raw packet capture. `AF_VSOCK` enables VM socket communication. | +| Risk if relaxed | The blocked syscalls support container escape (`mount`, `pivot_root`, `move_mount`, namespace creation), cross-process observation (`ptrace`, `process_vm_readv`, `pidfd_*`), raw kernel bypass (`bpf`, `io_uring_setup`, `perf_event_open`), and filter evasion (`seccomp`, `userfaultfd`). | | Recommendation | No action needed. OpenShell enforces this automatically. | ### Enforcement Application Order @@ -224,6 +228,7 @@ The agent never receives the provider API key. | Aspect | Detail | |---|---| | Default | Always active. The proxy handles `inference.local` before OPA policy evaluation. The gateway injects credentials on the host side. | +| Keep-alive isolation | If a sandbox reuses a keep-alive connection that previously carried a routed inference request for a subsequent non-inference request, the proxy denies the non-inference request with `connection not allowed by policy` and closes the connection. This prevents agents from reusing an inference-authorized connection for other destinations. | | What you can change | Configure inference routes with `openshell inference set`. | | Risk if bypassed | If an inference provider's host is added directly to `network_policies`, the agent could reach it with a stolen or hardcoded key, bypassing credential isolation. | | Recommendation | Do not add inference provider hosts to `network_policies`. Use OpenShell inference routing instead. | From 5c3015a6895dae9d394a71b0ff79d5989b9db5a0 Mon Sep 17 00:00:00 2001 From: Mrunal Patel Date: Fri, 17 Apr 2026 14:06:29 -0700 Subject: [PATCH 010/142] docs(contributing): add bash shell setup example for mise (#877) Signed-off-by: Mrunal Patel --- CONTRIBUTING.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dcb3f303fe..2852bfa438 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -107,6 +107,9 @@ After installing `mise`, activate it with `mise activate` or [add it to your she Shell setup examples: ```bash +# Bash +echo 'eval "$(~/.local/bin/mise activate bash)"' >> ~/.bashrc + # Fish echo '~/.local/bin/mise activate fish | source' >> ~/.config/fish/config.fish From ae7e901000da511d7ef942da55b35a3e5ed0cce0 Mon Sep 17 00:00:00 2001 From: mjamiv <142179942+mjamiv@users.noreply.github.com> Date: Fri, 17 Apr 2026 16:25:59 -0500 Subject: [PATCH 011/142] fix(sandbox): strip " (deleted)" suffix from unlinked /proc//exe paths (#844) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(sandbox): strip " (deleted)" suffix from unlinked /proc//exe paths When a running binary is unlinked from its filesystem path — the common case is a `docker cp` hot-swap of `/opt/openshell/bin/openshell-sandbox` during the `cluster-deploy-fast` dev upgrade workflow — the Linux kernel appends the literal string ` (deleted)` to the `/proc//exe` readlink target. The tainted `PathBuf` then flows into `collect_ancestor_binaries` and on into `BinaryIdentityCache::verify_or_cache`, which tries to `std::fs::metadata` the path. `stat()` fails with `ENOENT` because the literal suffix isn't a real filesystem path, and the CONNECT proxy denies every outbound request with: ancestor integrity check failed for \ /opt/openshell/bin/openshell-sandbox (deleted): \ Failed to stat ...: No such file or directory (os error 2) Reproduced in production 2026-04-15: a cluster-deploy-fast-style hot-swap of the supervisor binary caused every pod whose PID 1 held the now-deleted inode to deny ALL outbound CONNECTs (slack.com, registry.npmjs.org, 169.254.169.254, etc.), breaking Slack REST delivery, npm installs, and IMDS probes simultaneously. Existing pre-hot-swap TCP tunnels (e.g. Slack Socket Mode WSS) kept working because they never re-evaluate the proxy. Strip the suffix in `binary_path()` so downstream callers see the clean, grep-friendly path. This aligns the cache key and log messages with the original on-disk location. Note: stripping the suffix does NOT by itself make the identity cache tolerant of a legitimate binary replacement — `verify_or_cache` will now `stat` and hash whatever currently lives at the stripped path, which is the NEW binary, and surface a clearer `Binary integrity violation` error. Fully unblocking the cluster-deploy-fast hot-swap workflow needs a follow-up that either (a) reads running-binary content from `/proc//exe` directly via `File::open` (procfs resolves this to the live in-memory executable even when the original inode has been unlinked), or (b) keys the identity cache by exec dev+inode instead of path. Happy to send that as a separate PR once the approach is decided — filing this narrow fix first because it stands on its own: it fixes a concrete misleading error and unblocks the obvious next step. Added `binary_path_strips_deleted_suffix` test that copies `/bin/sleep` to a temp path, spawns a child from it, unlinks the temp binary, verifies the raw readlink contains the ` (deleted)` suffix, then asserts the public API returns the stripped path. Signed-off-by: mjamiv * fix(sandbox): narrow the " (deleted)" suffix strip and exercise it end-to-end Address review feedback from @johntmyers on the initial version of this PR: procfs::binary_path - Only strip the kernel's " (deleted)" suffix when stat() on the raw readlink target reports NotFound. A live executable whose basename literally ends with " (deleted)" is now returned unchanged instead of being silently truncated, which matters because identity.rs hashes whatever this function returns as a trust anchor. - Operate on raw bytes via OsStrExt, so filenames that are not valid UTF-8 still get exactly one suffix stripped. The previous strip_suffix-on-&str path skipped non-UTF-8 entirely and fell through to returning the tainted path. - Expand the doc comment to describe both guardrails. procfs tests - binary_path_preserves_live_deleted_basename: copy /bin/sleep to a live file literally named "sleepy (deleted)", spawn it, and assert that the returned path still ends with " (deleted)". - binary_path_strips_suffix_for_non_utf8_filename: exec a binary whose basename contains a 0xFF byte, unlink it, and assert that binary_path returns the stripped non-UTF-8 path. Writes the bytes with OpenOptions + sync_all + explicit drop so the write fd is fully released before exec() to avoid ETXTBSY under concurrent tests. proxy: extract resolve_process_identity helper - Pull the peer-resolution + TOFU verify + ancestor walk + cmdline collection block out of evaluate_opa_tcp into resolve_process_identity. - Introduce IdentityError which carries the deny reason along with whatever partial identity data was resolved before the failure so evaluate_opa_tcp can thread that into ConnectDecision unchanged. - evaluate_opa_tcp now calls the helper and proceeds straight to the OPA evaluate step; the surface visible to OPA and OCSF is unchanged. proxy: end-to-end regression test for the hot-swap contract - resolve_process_identity_surfaces_binary_integrity_violation_on_hot_swap stands up a real TcpListener, copies /bin/bash to a temp path, primes BinaryIdentityCache with that binary, spawns bash with a /dev/tcp one-liner that opens a real connection to the listener, and captures the peer's ephemeral port from accept(). - Simulates `docker cp` correctly: unlink the running binary (which persists via the child's exec mapping) and create a fresh file with different bytes at the same path. Writing in place is rejected by the kernel with ETXTBSY, so the old single-inode approach did not actually model the production failure mode. - Asserts the error returned by resolve_process_identity contains "Binary integrity violation" (from BinaryIdentityCache) and does NOT contain "Failed to stat" or "(deleted)" — the pre-PR-#844 failure mode. The binary field on the error is populated and is free of the tainted suffix. - Skips cleanly if /bin/bash is not installed. Child process is always reaped before the assertion block so a failure does not leak a sleeping process. --------- Signed-off-by: mjamiv --- crates/openshell-sandbox/src/procfs.rs | 215 ++++++++++++++++- crates/openshell-sandbox/src/proxy.rs | 311 +++++++++++++++++++++---- 2 files changed, 479 insertions(+), 47 deletions(-) diff --git a/crates/openshell-sandbox/src/procfs.rs b/crates/openshell-sandbox/src/procfs.rs index 785a9489ec..e488a8b51c 100644 --- a/crates/openshell-sandbox/src/procfs.rs +++ b/crates/openshell-sandbox/src/procfs.rs @@ -19,17 +19,63 @@ use tracing::debug; /// `/proc/{pid}/cmdline` because `argv[0]` is trivially spoofable by any /// process and must not be used as a trusted identity source. /// -/// If this fails, ensure the proxy process has permission to read -/// `/proc//exe` (e.g. same user, or `CAP_SYS_PTRACE`). +/// ### Unlinked binaries (`(deleted)` suffix) +/// +/// When a running binary is unlinked from its filesystem path — the common +/// case is a `docker cp` hot-swap of `/opt/openshell/bin/openshell-sandbox` +/// during a `cluster-deploy-fast` dev upgrade — the kernel appends the +/// literal string `" (deleted)"` to the `/proc//exe` readlink target. +/// The raw tainted path (e.g. `"/opt/openshell/bin/openshell-sandbox (deleted)"`) +/// is not a real filesystem path: any downstream `stat()` fails with `ENOENT`. +/// +/// We strip the suffix so callers see a clean, grep-friendly path suitable +/// for cache keys and log messages. The strip is guarded: we only strip when +/// `stat()` on the raw readlink target reports `NotFound`, so a live executable +/// whose basename literally ends with `" (deleted)"` is returned unchanged. +/// The comparison is done on raw bytes via `OsStrExt`, so filenames that are +/// not valid UTF-8 are still handled correctly. Exactly one kernel-added +/// suffix is stripped. +/// +/// This does NOT claim the file at the stripped path is the same binary that +/// the process is executing — the on-disk inode may now be arbitrary. Callers +/// that need to verify the running binary's *contents* (for integrity +/// checking) should read the magic `/proc//exe` symlink directly via +/// `File::open`, which procfs resolves to the live in-memory executable even +/// when the original inode has been unlinked. +/// +/// If the readlink itself fails, ensure the proxy process has permission +/// to read `/proc//exe` (e.g. same user, or `CAP_SYS_PTRACE`). #[cfg(target_os = "linux")] pub fn binary_path(pid: i32) -> Result { - std::fs::read_link(format!("/proc/{pid}/exe")).map_err(|e| { + use std::ffi::OsString; + use std::io::ErrorKind; + use std::os::unix::ffi::{OsStrExt, OsStringExt}; + + const DELETED_SUFFIX: &[u8] = b" (deleted)"; + + let link = format!("/proc/{pid}/exe"); + let target = std::fs::read_link(&link).map_err(|e| { miette::miette!( "Failed to read /proc/{pid}/exe: {e}. \ Cannot determine binary identity — denying request. \ Hint: the proxy may need CAP_SYS_PTRACE or to run as the same user." ) - }) + })?; + + // Only strip when the raw readlink target cannot be stat'd and its bytes + // end with the kernel-added suffix. This preserves live executables whose + // basename legitimately ends with " (deleted)" and handles non-UTF-8 + // filenames correctly. + let raw_target_missing = + matches!(std::fs::metadata(&target), Err(err) if err.kind() == ErrorKind::NotFound); + + let bytes = target.as_os_str().as_bytes(); + if raw_target_missing && bytes.ends_with(DELETED_SUFFIX) { + let stripped = bytes[..bytes.len() - DELETED_SUFFIX.len()].to_vec(); + return Ok(PathBuf::from(OsString::from_vec(stripped))); + } + + Ok(target) } /// Resolve the binary path of the TCP peer inside a sandbox network namespace. @@ -391,6 +437,167 @@ mod tests { assert!(path.exists()); } + /// Verify that an unlinked binary's path is returned without the + /// kernel's " (deleted)" suffix. This is the common case during a + /// `docker cp` hot-swap of the supervisor binary — before this strip, + /// callers that `stat()` the returned path get `ENOENT` and the + /// ancestor integrity check in the CONNECT proxy denies every request. + #[cfg(target_os = "linux")] + #[test] + fn binary_path_strips_deleted_suffix() { + use std::os::unix::fs::PermissionsExt; + + // Copy /bin/sleep to a temp path we control so we can unlink it. + let tmp = tempfile::TempDir::new().unwrap(); + let exe_path = tmp.path().join("deleted-sleep"); + std::fs::copy("/bin/sleep", &exe_path).unwrap(); + std::fs::set_permissions(&exe_path, std::fs::Permissions::from_mode(0o755)).unwrap(); + + // Spawn a child from the temp binary, then unlink it while the + // child is still running. The child keeps the exec mapping via + // `/proc//exe`, but readlink will now return the tainted + // " (deleted)" string. + let mut child = std::process::Command::new(&exe_path) + .arg("5") + .spawn() + .unwrap(); + let pid: i32 = child.id().cast_signed(); + std::fs::remove_file(&exe_path).unwrap(); + + // Sanity check: the raw readlink should contain " (deleted)". + let raw = std::fs::read_link(format!("/proc/{pid}/exe")) + .unwrap() + .to_string_lossy() + .into_owned(); + assert!( + raw.ends_with(" (deleted)"), + "kernel should append ' (deleted)' to unlinked exe readlink; got {raw:?}" + ); + + // The public API should return the stripped path, not the tainted one. + let resolved = binary_path(pid).expect("binary_path should succeed for deleted binary"); + assert_eq!( + resolved, exe_path, + "binary_path should strip the ' (deleted)' suffix" + ); + let resolved_str = resolved.to_string_lossy(); + assert!( + !resolved_str.contains("(deleted)"), + "stripped path must not contain '(deleted)'; got {resolved_str:?}" + ); + + let _ = child.kill(); + let _ = child.wait(); + } + + /// A live executable whose basename literally ends with `" (deleted)"` + /// must be returned unchanged — we only strip when `stat()` reports + /// the raw readlink target missing. This guards against the trusted + /// identity source misattributing a running binary to a truncated + /// sibling path. + #[cfg(target_os = "linux")] + #[test] + fn binary_path_preserves_live_deleted_basename() { + use std::os::unix::fs::PermissionsExt; + + let tmp = tempfile::TempDir::new().unwrap(); + // Basename literally ends with " (deleted)" while the file is still + // on disk — a pathological but legal filename. + let exe_path = tmp.path().join("sleepy (deleted)"); + std::fs::copy("/bin/sleep", &exe_path).unwrap(); + std::fs::set_permissions(&exe_path, std::fs::Permissions::from_mode(0o755)).unwrap(); + + let mut child = std::process::Command::new(&exe_path) + .arg("5") + .spawn() + .unwrap(); + let pid: i32 = child.id().cast_signed(); + + // File is still linked — binary_path must return the path unchanged, + // suffix and all. + let resolved = binary_path(pid).expect("binary_path should succeed for live binary"); + assert_eq!( + resolved, exe_path, + "binary_path must NOT strip ' (deleted)' from a live executable's basename" + ); + assert!( + resolved.to_string_lossy().ends_with(" (deleted)"), + "stripped path unexpectedly trimmed a real filename: {resolved:?}" + ); + + let _ = child.kill(); + let _ = child.wait(); + } + + /// An unlinked executable whose filename contains non-UTF-8 bytes must + /// still strip exactly one kernel-added `" (deleted)"` suffix. We operate + /// on raw bytes via `OsStrExt`, so invalid UTF-8 is not a reason to skip + /// the strip and return a path that downstream `stat()` calls will reject. + #[cfg(target_os = "linux")] + #[test] + fn binary_path_strips_suffix_for_non_utf8_filename() { + use std::ffi::OsString; + use std::io::Write; + use std::os::unix::ffi::{OsStrExt, OsStringExt}; + use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; + + let tmp = tempfile::TempDir::new().unwrap(); + // 0xFF is not valid UTF-8. Build the filename on raw bytes. + let mut raw_name: Vec = b"badname-".to_vec(); + raw_name.push(0xFF); + raw_name.extend_from_slice(b".bin"); + let exe_path = tmp.path().join(OsString::from_vec(raw_name)); + + // Write bytes explicitly (instead of `std::fs::copy`) with an + // explicit `sync_all()` + scope drop so the write fd is fully closed + // before we `exec()` the file. Otherwise concurrent tests can race + // the kernel into returning ETXTBSY on spawn. + let bytes = std::fs::read("/bin/sleep").expect("read /bin/sleep"); + { + let mut f = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o755) + .open(&exe_path) + .expect("create non-UTF-8 target file"); + f.write_all(&bytes).expect("write bytes"); + f.sync_all().expect("sync_all before exec"); + } + + let mut child = std::process::Command::new(&exe_path) + .arg("5") + .spawn() + .unwrap(); + let pid: i32 = child.id().cast_signed(); + std::fs::remove_file(&exe_path).unwrap(); + + // Sanity: raw readlink ends with " (deleted)" and is not valid UTF-8. + let raw = std::fs::read_link(format!("/proc/{pid}/exe")).unwrap(); + let raw_bytes = raw.as_os_str().as_bytes(); + assert!( + raw_bytes.ends_with(b" (deleted)"), + "kernel should append ' (deleted)' to unlinked exe readlink" + ); + assert!( + std::str::from_utf8(raw_bytes).is_err(), + "test precondition: raw readlink must contain non-UTF-8 bytes" + ); + + let resolved = + binary_path(pid).expect("binary_path should succeed for non-UTF-8 unlinked path"); + assert_eq!( + resolved, exe_path, + "binary_path must strip exactly one ' (deleted)' suffix for non-UTF-8 paths" + ); + assert!( + !resolved.as_os_str().as_bytes().ends_with(b" (deleted)"), + "stripped path must not end with ' (deleted)'" + ); + + let _ = child.kill(); + let _ = child.wait(); + } + #[cfg(target_os = "linux")] #[test] fn collect_descendants_includes_self() { diff --git a/crates/openshell-sandbox/src/proxy.rs b/crates/openshell-sandbox/src/proxy.rs index f91f2c551f..01ff303477 100644 --- a/crates/openshell-sandbox/src/proxy.rs +++ b/crates/openshell-sandbox/src/proxy.rs @@ -885,6 +885,98 @@ async fn handle_tcp_connection( Ok(()) } +/// Resolved process identity for a TCP peer: binary path, PID, ancestor chain, +/// cmdline paths, and the TOFU-verified binary hash. +/// +/// Produced by [`resolve_process_identity`]; consumed by [`evaluate_opa_tcp`] +/// and by the identity-chain regression tests. +#[cfg(target_os = "linux")] +struct ResolvedIdentity { + bin_path: PathBuf, + binary_pid: u32, + ancestors: Vec, + cmdline_paths: Vec, + bin_hash: String, +} + +/// Error from [`resolve_process_identity`]. Carries the deny reason and +/// whatever partial identity data was resolved before the failure so the +/// caller can include it in the [`ConnectDecision`] and OCSF event. +#[cfg(target_os = "linux")] +struct IdentityError { + reason: String, + binary: Option, + binary_pid: Option, + ancestors: Vec, +} + +/// Resolve the identity of the process owning a TCP peer connection. +/// +/// Walks `/proc//net/tcp` to find the socket inode, locates +/// the owning PID, reads `/proc//exe`, TOFU-verifies the binary hash, +/// walks the ancestor chain verifying each one, and collects cmdline-derived +/// absolute paths for script detection. +/// +/// This is the identity-resolution block of [`evaluate_opa_tcp`] extracted +/// into a standalone helper so it can be exercised by Linux-only regression +/// tests without a full OPA engine. The key invariant under test is that on +/// a hot-swap of the peer binary, the failure mode is +/// `"Binary integrity violation"` (from the identity cache) rather than +/// `"Failed to stat ... (deleted)"` (from the kernel-tainted path). +#[cfg(target_os = "linux")] +fn resolve_process_identity( + entrypoint_pid: u32, + peer_port: u16, + identity_cache: &BinaryIdentityCache, +) -> std::result::Result { + let (bin_path, binary_pid) = + crate::procfs::resolve_tcp_peer_identity(entrypoint_pid, peer_port).map_err(|e| { + IdentityError { + reason: format!("failed to resolve peer binary: {e}"), + binary: None, + binary_pid: None, + ancestors: vec![], + } + })?; + + let bin_hash = identity_cache + .verify_or_cache(&bin_path) + .map_err(|e| IdentityError { + reason: format!("binary integrity check failed: {e}"), + binary: Some(bin_path.clone()), + binary_pid: Some(binary_pid), + ancestors: vec![], + })?; + + let ancestors = crate::procfs::collect_ancestor_binaries(binary_pid, entrypoint_pid); + + for ancestor in &ancestors { + identity_cache + .verify_or_cache(ancestor) + .map_err(|e| IdentityError { + reason: format!( + "ancestor integrity check failed for {}: {e}", + ancestor.display() + ), + binary: Some(bin_path.clone()), + binary_pid: Some(binary_pid), + ancestors: ancestors.clone(), + })?; + } + + let mut exclude = ancestors.clone(); + exclude.push(bin_path.clone()); + let cmdline_paths = crate::procfs::collect_cmdline_paths(binary_pid, entrypoint_pid, &exclude); + + Ok(ResolvedIdentity { + bin_path, + binary_pid, + ancestors, + cmdline_paths, + bin_hash, + }) +} + /// Evaluate OPA policy for a TCP connection with identity binding via /proc/net/tcp. #[cfg(target_os = "linux")] fn evaluate_opa_tcp( @@ -927,55 +1019,26 @@ fn evaluate_opa_tcp( let total_start = std::time::Instant::now(); let peer_port = peer_addr.port(); - let (bin_path, binary_pid) = match crate::procfs::resolve_tcp_peer_identity(pid, peer_port) { - Ok(r) => r, - Err(e) => { + let identity = match resolve_process_identity(pid, peer_port, identity_cache) { + Ok(id) => id, + Err(err) => { return deny( - format!("failed to resolve peer binary: {e}"), - None, - None, - vec![], + err.reason, + err.binary, + err.binary_pid, + err.ancestors, vec![], ); } }; - // TOFU verify the immediate binary - let bin_hash = match identity_cache.verify_or_cache(&bin_path) { - Ok(h) => h, - Err(e) => { - return deny( - format!("binary integrity check failed: {e}"), - Some(bin_path), - Some(binary_pid), - vec![], - vec![], - ); - } - }; - - // Walk the process tree upward to collect ancestor binaries - let ancestors = crate::procfs::collect_ancestor_binaries(binary_pid, pid); - - for ancestor in &ancestors { - if let Err(e) = identity_cache.verify_or_cache(ancestor) { - return deny( - format!( - "ancestor integrity check failed for {}: {e}", - ancestor.display() - ), - Some(bin_path), - Some(binary_pid), - ancestors.clone(), - vec![], - ); - } - } - - // Collect cmdline paths for script-based binary detection. - let mut exclude = ancestors.clone(); - exclude.push(bin_path.clone()); - let cmdline_paths = crate::procfs::collect_cmdline_paths(binary_pid, pid, &exclude); + let ResolvedIdentity { + bin_path, + binary_pid, + ancestors, + cmdline_paths, + bin_hash, + } = identity; let input = NetworkInput { host: host.to_string(), @@ -3735,4 +3798,166 @@ mod tests { let body_start = resp_str.find("\r\n\r\n").unwrap() + 4; assert_eq!(resp_str[body_start..].len(), cl); } + + /// End-to-end regression for the `docker cp` hot-swap hazard that + /// motivated `binary_path()` stripping the kernel's `" (deleted)"` + /// suffix (PR #844). + /// + /// Before the strip, the identity-resolution chain inside + /// `evaluate_opa_tcp` failed with `"Failed to stat + /// /opt/openshell/bin/openshell-sandbox (deleted)"` because + /// `BinaryIdentityCache::verify_or_cache()` tried to `metadata()` the + /// tainted path. That masked the real security signal: a live process + /// was now bound to a *different* binary on disk than the one that was + /// TOFU-cached. After the strip, `binary_path()` returns a path that + /// stats fine, the cache rehashes the new bytes, and the hash mismatch + /// surfaces as a `Binary integrity violation` error — the contract this + /// PR is trying to establish. + /// + /// Test shape (from the review comment on the initial PR): + /// 1. Start a `TcpListener` in the test process. + /// 2. Copy `/bin/bash` to a temp path we control. + /// 3. Prime `BinaryIdentityCache` with that temp binary's hash. + /// 4. Spawn the temp bash as a child with a `/dev/tcp` one-liner that + /// opens a real TCP connection to the listener and holds it open. + /// 5. Accept the connection on the listener side and capture the peer's + /// ephemeral port — that's what `resolve_process_identity` uses to + /// walk `/proc/net/tcp` back to the child PID. + /// 6. Overwrite the temp bash on disk with different bytes to simulate + /// a `docker cp` hot-swap. The running child is unaffected (it still + /// executes from its in-memory image), but `/proc//exe` will + /// now readlink to `" (deleted)"` OR the overwritten file, depending + /// on whether the filesystem reused the inode. + /// 7. Call `resolve_process_identity` and assert: + /// - the error reason contains `"Binary integrity violation"` (the + /// cache detected the tampered on-disk bytes), and + /// - the error reason does NOT contain `"Failed to stat"` or + /// `"(deleted)"` (the old pre-strip failure mode). + #[cfg(target_os = "linux")] + #[test] + fn resolve_process_identity_surfaces_binary_integrity_violation_on_hot_swap() { + use crate::identity::BinaryIdentityCache; + use std::io::Read; + use std::net::TcpListener; + use std::os::unix::fs::PermissionsExt; + use std::process::{Command, Stdio}; + use std::time::Duration; + + // Skip if /bin/bash is not present (e.g. minimal containers). + if !std::path::Path::new("/bin/bash").exists() { + eprintln!("skipping: /bin/bash not available"); + return; + } + + // 1. Start a listener on loopback. + let listener = TcpListener::bind("127.0.0.1:0").expect("bind"); + let listener_port = listener.local_addr().unwrap().port(); + + // 2. Copy /bin/bash to a temp path. + let tmp = tempfile::TempDir::new().unwrap(); + let bash_v1 = tmp.path().join("hotswap-bash"); + std::fs::copy("/bin/bash", &bash_v1).expect("copy bash"); + std::fs::set_permissions(&bash_v1, std::fs::Permissions::from_mode(0o755)).unwrap(); + + // 3. Prime the cache with the v1 hash of the temp bash. + let cache = BinaryIdentityCache::new(); + let v1_hash = cache + .verify_or_cache(&bash_v1) + .expect("prime cache with v1 bash hash"); + assert!(!v1_hash.is_empty()); + + // 4. Spawn the temp bash with a /dev/tcp one-liner that opens a real + // connection to the listener and sleeps to keep it open. The + // `read -t` blocks on stdin so the shell stays resident. + let script = format!("exec 3<>/dev/tcp/127.0.0.1/{listener_port}; sleep 30 <&3"); + let mut child = Command::new(&bash_v1) + .arg("-c") + .arg(&script) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn hotswap-bash child"); + + // 5. Accept on the listener side, capture the peer port. + listener.set_nonblocking(false).expect("blocking listener"); + let (mut stream, peer_addr) = match listener.accept() { + Ok(pair) => pair, + Err(e) => { + let _ = child.kill(); + let _ = child.wait(); + panic!("failed to accept child connection: {e}"); + } + }; + let peer_port = peer_addr.port(); + // Drain any spurious data; we just need the socket open. + stream + .set_read_timeout(Some(Duration::from_millis(50))) + .ok(); + let mut buf = [0u8; 16]; + let _ = stream.read(&mut buf); + + // Give the kernel a moment so /proc//net/tcp and + // /proc//fd/ both reflect the ESTABLISHED socket. + std::thread::sleep(Duration::from_millis(50)); + + // 6. Simulate `docker cp`: unlink the running binary and create a + // fresh file with different bytes at the same path. Writing + // in place via O_TRUNC is rejected by the kernel with ETXTBSY + // because the inode is still being executed. Unlink is cheap: + // the inode persists in memory via the child's exec mapping, + // so the child keeps running, but a new inode now lives at + // `bash_v1` with a different SHA-256. + std::fs::remove_file(&bash_v1).expect("unlink running bash_v1"); + let tampered_bytes = b"#!/bin/sh\n# tampered bash v2 from hotswap test\nexit 0\n"; + std::fs::write(&bash_v1, tampered_bytes).expect("write replacement bytes"); + + // 7. Resolve identity through the real helper and assert the + // contract: we want "Binary integrity violation", not + // "Failed to stat ... (deleted)". + let test_pid = std::process::id(); + let result = resolve_process_identity(test_pid, peer_port, &cache); + + // Always clean up the child before asserting so a failure doesn't + // leak a sleeping process across test runs. + let _ = child.kill(); + let _ = child.wait(); + + match result { + Ok(_) => panic!( + "resolve_process_identity unexpectedly succeeded after hot-swap; \ + the cache should have detected the tampered on-disk bytes" + ), + Err(err) => { + assert!( + err.reason.contains("Binary integrity violation"), + "expected 'Binary integrity violation' error, got: {}", + err.reason + ); + assert!( + !err.reason.contains("Failed to stat"), + "pre-PR-#844 failure mode leaked: {}", + err.reason + ); + assert!( + !err.reason.contains("(deleted)"), + "resolved path still contains '(deleted)' suffix: {}", + err.reason + ); + // The binary field should be populated — we did resolve a + // path before failing. + assert!( + err.binary.is_some(), + "expected resolved binary path on integrity failure" + ); + if let Some(path) = &err.binary { + assert!( + !path.to_string_lossy().contains("(deleted)"), + "resolved binary path still tainted: {}", + path.display() + ); + } + } + } + } } From e39bb380409dc36092ab80258547225830a7b06d Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Fri, 17 Apr 2026 17:47:37 -0700 Subject: [PATCH 012/142] test(sandbox): fix flaky arm64 procfs binary_path tests (#881) --- crates/openshell-sandbox/src/procfs.rs | 90 +++++++++++++++++--------- 1 file changed, 61 insertions(+), 29 deletions(-) diff --git a/crates/openshell-sandbox/src/procfs.rs b/crates/openshell-sandbox/src/procfs.rs index e488a8b51c..a6dd379b89 100644 --- a/crates/openshell-sandbox/src/procfs.rs +++ b/crates/openshell-sandbox/src/procfs.rs @@ -399,6 +399,52 @@ mod tests { use super::*; use std::io::Write; + /// Block until `/proc//exe` points at `target`. `Command::spawn` returns + /// once the child is scheduled, not once it has completed `exec()`; on + /// contended runners the readlink can still show the parent (test harness) + /// binary for a brief window. Byte-level `starts_with` tolerates the kernel's + /// `" (deleted)"` suffix on unlinked executables. + #[cfg(target_os = "linux")] + fn wait_for_child_exec(pid: i32, target: &std::path::Path) { + use std::os::unix::ffi::OsStrExt as _; + let target_bytes = target.as_os_str().as_bytes(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2); + loop { + if let Ok(link) = std::fs::read_link(format!("/proc/{pid}/exe")) + && link.as_os_str().as_bytes().starts_with(target_bytes) + { + return; + } + assert!( + std::time::Instant::now() < deadline, + "child pid {pid} did not exec into {target:?} within 2s" + ); + std::thread::sleep(std::time::Duration::from_millis(10)); + } + } + + /// Retry `Command::spawn` on `ETXTBSY`. The kernel rejects `execve` when + /// `inode->i_writecount > 0`, and the release of that counter after the + /// writer fd is closed isn't synchronous with `close(2)` under contention — + /// so the very-next-instruction `execve` can still race it. Any other error + /// surfaces immediately. + #[cfg(target_os = "linux")] + fn spawn_retrying_on_etxtbsy(cmd: &mut std::process::Command) -> std::process::Child { + let mut attempts = 0; + loop { + match cmd.spawn() { + Ok(child) => return child, + Err(err) + if err.kind() == std::io::ErrorKind::ExecutableFileBusy && attempts < 20 => + { + attempts += 1; + std::thread::sleep(std::time::Duration::from_millis(50)); + } + Err(err) => panic!("spawn failed after {attempts} ETXTBSY retries: {err}"), + } + } + } + #[test] fn file_sha256_computes_correct_hash() { let mut tmp = tempfile::NamedTempFile::new().unwrap(); @@ -457,11 +503,11 @@ mod tests { // child is still running. The child keeps the exec mapping via // `/proc//exe`, but readlink will now return the tainted // " (deleted)" string. - let mut child = std::process::Command::new(&exe_path) - .arg("5") - .spawn() - .unwrap(); + let mut cmd = std::process::Command::new(&exe_path); + cmd.arg("5"); + let mut child = spawn_retrying_on_etxtbsy(&mut cmd); let pid: i32 = child.id().cast_signed(); + wait_for_child_exec(pid, &exe_path); std::fs::remove_file(&exe_path).unwrap(); // Sanity check: the raw readlink should contain " (deleted)". @@ -507,11 +553,11 @@ mod tests { std::fs::copy("/bin/sleep", &exe_path).unwrap(); std::fs::set_permissions(&exe_path, std::fs::Permissions::from_mode(0o755)).unwrap(); - let mut child = std::process::Command::new(&exe_path) - .arg("5") - .spawn() - .unwrap(); + let mut cmd = std::process::Command::new(&exe_path); + cmd.arg("5"); + let mut child = spawn_retrying_on_etxtbsy(&mut cmd); let pid: i32 = child.id().cast_signed(); + wait_for_child_exec(pid, &exe_path); // File is still linked — binary_path must return the path unchanged, // suffix and all. @@ -537,9 +583,8 @@ mod tests { #[test] fn binary_path_strips_suffix_for_non_utf8_filename() { use std::ffi::OsString; - use std::io::Write; use std::os::unix::ffi::{OsStrExt, OsStringExt}; - use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; + use std::os::unix::fs::PermissionsExt; let tmp = tempfile::TempDir::new().unwrap(); // 0xFF is not valid UTF-8. Build the filename on raw bytes. @@ -548,27 +593,14 @@ mod tests { raw_name.extend_from_slice(b".bin"); let exe_path = tmp.path().join(OsString::from_vec(raw_name)); - // Write bytes explicitly (instead of `std::fs::copy`) with an - // explicit `sync_all()` + scope drop so the write fd is fully closed - // before we `exec()` the file. Otherwise concurrent tests can race - // the kernel into returning ETXTBSY on spawn. - let bytes = std::fs::read("/bin/sleep").expect("read /bin/sleep"); - { - let mut f = std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .mode(0o755) - .open(&exe_path) - .expect("create non-UTF-8 target file"); - f.write_all(&bytes).expect("write bytes"); - f.sync_all().expect("sync_all before exec"); - } + std::fs::copy("/bin/sleep", &exe_path).unwrap(); + std::fs::set_permissions(&exe_path, std::fs::Permissions::from_mode(0o755)).unwrap(); - let mut child = std::process::Command::new(&exe_path) - .arg("5") - .spawn() - .unwrap(); + let mut cmd = std::process::Command::new(&exe_path); + cmd.arg("5"); + let mut child = spawn_retrying_on_etxtbsy(&mut cmd); let pid: i32 = child.id().cast_signed(); + wait_for_child_exec(pid, &exe_path); std::fs::remove_file(&exe_path).unwrap(); // Sanity: raw readlink ends with " (deleted)" and is not valid UTF-8. From 40e9bf6feb7202625580e50863ad7abf456f97af Mon Sep 17 00:00:00 2001 From: "John T. Myers" <9696606+johntmyers@users.noreply.github.com> Date: Mon, 20 Apr 2026 08:00:02 -0700 Subject: [PATCH 013/142] feat(policy): add incremental sandbox policy updates (#860) * feat(policy): add incremental sandbox policy updates Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> * docs(policies): expand incremental update guidance Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> * feat(policy): audit incremental updates in gateway logs * docs(policy): quote glob specs in shell examples --------- Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- .../skills/debug-openshell-cluster/SKILL.md | 2 +- .agents/skills/openshell-cli/SKILL.md | 13 +- .agents/skills/openshell-cli/cli-reference.md | 25 +- Cargo.lock | 1 + architecture/build-containers.md | 2 +- architecture/security-policy.md | 34 +- crates/openshell-cli/src/lib.rs | 1 + crates/openshell-cli/src/main.rs | 81 ++ crates/openshell-cli/src/policy_update.rs | 473 ++++++++ crates/openshell-cli/src/run.rs | 211 +++- crates/openshell-policy/src/lib.rs | 7 + crates/openshell-policy/src/merge.rs | 1016 +++++++++++++++++ crates/openshell-sandbox/src/grpc_client.rs | 1 + crates/openshell-server/Cargo.toml | 1 + crates/openshell-server/src/grpc/policy.rs | 914 ++++++++++++--- .../openshell-server/src/grpc/validation.rs | 7 + crates/openshell-server/src/tracing_bus.rs | 18 +- crates/openshell-tui/src/lib.rs | 4 + deploy/docker/Dockerfile.images | 1 + docs/observability/accessing-logs.mdx | 2 + docs/reference/policy-schema.mdx | 2 +- docs/sandboxes/policies.mdx | 257 ++++- docs/security/best-practices.mdx | 6 +- proto/openshell.proto | 45 + tasks/scripts/cluster-deploy-fast.sh | 4 +- 25 files changed, 2923 insertions(+), 205 deletions(-) create mode 100644 crates/openshell-cli/src/policy_update.rs create mode 100644 crates/openshell-policy/src/merge.rs diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index f4c5672a20..3d2e6f7827 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -182,7 +182,7 @@ Component images (server, sandbox) can reach kubelet via two paths: **Local/external pull mode** (default local via `mise run cluster`): Local images are tagged to the configured local registry base (default `127.0.0.1:5000/openshell/*`), pushed to that registry, and pulled by k3s via `registries.yaml` mirror endpoint (typically `host.docker.internal:5000`). The `cluster` task pushes prebuilt local tags (`openshell/*:dev`, falling back to `localhost:5000/openshell/*:dev` or `127.0.0.1:5000/openshell/*:dev`). -Gateway image builds now stage a partial Rust workspace from `deploy/docker/Dockerfile.images`. If cargo fails with a missing manifest under `/build/crates/...`, verify that every current gateway dependency crate (including `openshell-driver-kubernetes`) is copied into the staged workspace there. +Gateway image builds now stage a partial Rust workspace from `deploy/docker/Dockerfile.images`. If cargo fails with a missing manifest under `/build/crates/...`, or an imported symbol exists locally but is missing in the image build, verify that every current gateway dependency crate (including `openshell-driver-kubernetes` and `openshell-ocsf`) is copied into the staged workspace there. ```bash # Verify image refs currently used by openshell deployment diff --git a/.agents/skills/openshell-cli/SKILL.md b/.agents/skills/openshell-cli/SKILL.md index 132c996861..d639e82b59 100644 --- a/.agents/skills/openshell-cli/SKILL.md +++ b/.agents/skills/openshell-cli/SKILL.md @@ -421,10 +421,14 @@ Watch for `deny` actions that indicate the user's work is being blocked by polic When denied actions are observed: -1. Pull current policy: `openshell policy get work-session --full > policy.yaml` -2. Modify the policy to allow the blocked actions (use `generate-sandbox-policy` skill for content) -3. Push the update: `openshell policy set work-session --policy policy.yaml --wait` -4. Verify: `openshell policy list work-session` +1. Prefer incremental updates for additive network changes: + `openshell policy update work-session --add-endpoint api.github.com:443:read-only:rest:enforce --binary /usr/bin/gh --wait` + `openshell policy update work-session --add-allow 'api.github.com:443:POST:/repos/*/issues' --wait` +2. Use full YAML replacement when the change is broad or touches non-network fields: + `openshell policy get work-session --full > policy.yaml` + Modify the policy to allow the blocked actions (use `generate-sandbox-policy` skill for content) + `openshell policy set work-session --policy policy.yaml --wait` +3. Verify: `openshell policy list work-session` The user does not need to disconnect -- policy updates are hot-reloaded within ~30 seconds (or immediately when using `--wait`, which polls for confirmation). @@ -543,6 +547,7 @@ $ openshell sandbox upload --help | Create with custom policy | `openshell sandbox create --policy ./p.yaml` | | Connect to sandbox | `openshell sandbox connect ` | | Stream live logs | `openshell logs --tail` | +| Incremental policy update | `openshell policy update --add-endpoint host:443:read-only:rest:enforce --binary /usr/bin/curl --wait` | | Pull current policy | `openshell policy get --full > p.yaml` | | Push updated policy | `openshell policy set --policy p.yaml --wait` | | Policy revision history | `openshell policy list ` | diff --git a/.agents/skills/openshell-cli/cli-reference.md b/.agents/skills/openshell-cli/cli-reference.md index e344f20dfd..c9c5450a00 100644 --- a/.agents/skills/openshell-cli/cli-reference.md +++ b/.agents/skills/openshell-cli/cli-reference.md @@ -268,9 +268,32 @@ View sandbox logs. Supports one-shot and streaming. ## Policy Commands +### `openshell policy update ` + +Incrementally merge live network policy changes into the current sandbox policy. Multiple flags in one invocation are applied as one atomic batch and create at most one new revision. + +| Flag | Default | Description | +|------|---------|-------------| +| `--add-endpoint ` | repeatable | `host:port[:access[:protocol[:enforcement]]]`. Adds or merges an endpoint. `access`: `read-only`, `read-write`, `full`. `protocol`: `rest`, `sql`. `enforcement`: `enforce`, `audit`. | +| `--remove-endpoint ` | repeatable | `host:port`. Removes the endpoint or just the requested port from a multi-port endpoint. | +| `--add-allow ` | repeatable | `host:port:METHOD:path_glob`. Adds REST allow rules to an existing `protocol: rest` endpoint. | +| `--add-deny ` | repeatable | `host:port:METHOD:path_glob`. Adds REST deny rules to an existing `protocol: rest` endpoint that already has an allow base. | +| `--remove-rule ` | repeatable | Deletes a named network rule. | +| `--binary ` | repeatable | Adds binaries to each `--add-endpoint` rule. Valid only with `--add-endpoint`. | +| `--rule-name ` | none | Overrides the generated rule name. Valid only when exactly one `--add-endpoint` is provided. | +| `--dry-run` | false | Preview the merged policy locally without sending an update to the gateway. | +| `--wait` | false | Wait for the sandbox to confirm the new policy revision is loaded. | +| `--timeout ` | 60 | Timeout for `--wait`. | + +Notes: + +- `--add-allow` and `--add-deny` currently operate only on `protocol: rest` endpoints. +- `--wait` cannot be combined with `--dry-run`. +- Use `policy set` when replacing the full policy or changing static sections. + ### `openshell policy set --policy ` -Update the policy on a live sandbox. Only the dynamic `network_policies` field can be changed at runtime. +Replace the full policy on a live sandbox. Only the dynamic `network_policies` field can be changed at runtime. | Flag | Default | Description | |------|---------|-------------| diff --git a/Cargo.lock b/Cargo.lock index 4b29a0c7f0..d5de42fb36 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3240,6 +3240,7 @@ dependencies = [ "miette", "openshell-core", "openshell-driver-kubernetes", + "openshell-ocsf", "openshell-policy", "openshell-router", "petname", diff --git a/architecture/build-containers.md b/architecture/build-containers.md index 196663e7a3..e619534b2b 100644 --- a/architecture/build-containers.md +++ b/architecture/build-containers.md @@ -63,7 +63,7 @@ The incremental deploy (`cluster-deploy-fast.sh`) fingerprints local Git changes | Changed files | Rebuild triggered | |---|---| | Cargo manifests, proto definitions, cross-build script | Gateway + supervisor | -| `crates/openshell-server/*`, `deploy/docker/Dockerfile.images` | Gateway | +| `crates/openshell-server/*`, `crates/openshell-ocsf/*`, `deploy/docker/Dockerfile.images` | Gateway | | `crates/openshell-sandbox/*`, `crates/openshell-policy/*` | Supervisor | | `deploy/helm/openshell/*` | Helm upgrade | diff --git a/architecture/security-policy.md b/architecture/security-policy.md index b8bda8f91a..82f0cc1dfa 100644 --- a/architecture/security-policy.md +++ b/architecture/security-policy.md @@ -162,6 +162,24 @@ This guarantees that the same logical policy always produces the same hash regar **Idempotent updates**: `UpdateSandboxPolicy` compares the deterministic hash of the submitted policy against the latest stored revision's hash. If they match, the handler returns the existing version and hash without creating a new revision. The CLI detects this (the returned version equals the pre-call version) and prints `Policy unchanged` instead of `Policy version N submitted`. This makes repeated `policy set` calls safe and idempotent. +### Incremental Merge Updates + +`UpdateConfigRequest.merge_operations` supports batched incremental changes to the dynamic `network_policies` section. The CLI exposes this as `openshell policy update`. + +Supported first-pass operations: + +- `--add-endpoint host:port[:access[:protocol[:enforcement]]]` +- `--remove-endpoint host:port` +- `--remove-rule ` +- `--add-allow host:port:METHOD:path_glob` +- `--add-deny host:port:METHOD:path_glob` + +`--add-allow` and `--add-deny` target existing `protocol: rest` endpoints only. `--binary` may be repeated with `--add-endpoint`, and `--rule-name` is allowed only when exactly one `--add-endpoint` is present. + +Each `openshell policy update` invocation is atomic at the revision level: the CLI sends one `merge_operations` batch, the server merges the whole batch into the latest policy, validates the result, and persists at most one new revision. Concurrency is handled with optimistic retries on the `(sandbox_id, version)` uniqueness boundary. If another writer wins first, the server refetches the latest policy, reapplies the full batch, revalidates it, and retries. This preserves batch atomicity without serializing all sandbox policy writes behind a sandbox-global mutex. + +The gateway emits per-sandbox OCSF `CONFIG:*` audit lines when incremental merge operations are applied and when draft chunks are approved or removed. These audit lines are streamed through the existing gateway log path, so operators can inspect the exact logical mutation that produced a policy revision without waiting for the sandbox poll loop to reload that revision. + ### Policy Revision Statuses | Status | Meaning | @@ -206,9 +224,20 @@ Failure scenarios that trigger LKG behavior include: ### CLI Commands -The `openshell policy` subcommand group manages live policy updates: +The `openshell policy` subcommand group manages live policy updates through full replacement (`policy set`) and incremental merges (`policy update`): ```bash +# Merge endpoint/rule changes into the current sandbox policy +openshell policy update \ + --add-endpoint api.github.com:443:read-only:rest:enforce \ + --binary /usr/bin/gh \ + --wait + +# Add a REST allow rule to an existing endpoint +openshell policy update \ + --add-allow api.github.com:443:POST:/repos/*/issues \ + --wait + # Push a new policy to a running sandbox openshell policy set --policy updated-policy.yaml @@ -255,6 +284,7 @@ Both `set` and `delete` require interactive confirmation (or `--yes` to bypass). When a global policy is active, sandbox-scoped policy mutations are blocked: - `policy set ` returns `FailedPrecondition: "policy is managed globally"` +- `policy update ` returns `FailedPrecondition: "policy is managed globally"` - `rule approve`, `rule approve-all` return `FailedPrecondition: "cannot approve rules while a global policy is active"` - Revoking a previously approved draft chunk is blocked (it would modify the sandbox policy) - Rejecting pending chunks is allowed (does not modify the sandbox policy) @@ -270,7 +300,7 @@ See [Gateway Settings Channel](gateway-settings.md#global-policy-lifecycle) for When `--full` is specified, the server includes the deserialized `SandboxPolicy` protobuf in the `SandboxPolicyRevision.policy` field (see `crates/openshell-server/src/grpc.rs` -- `policy_record_to_revision()` with `include_policy: true`). The CLI converts this proto back to YAML via `policy_to_yaml()`, which uses a `BTreeMap` for `network_policies` to produce deterministic key ordering. See `crates/openshell-cli/src/run.rs` -- `policy_to_yaml()`, `policy_get()`. -See `crates/openshell-cli/src/main.rs` -- `PolicyCommands` enum, `crates/openshell-cli/src/run.rs` -- `policy_set()`, `policy_get()`, `policy_list()`. +See `crates/openshell-cli/src/main.rs` -- `PolicyCommands` enum, `crates/openshell-cli/src/run.rs` -- `policy_update()`, `policy_set()`, `policy_get()`, `policy_list()`. --- diff --git a/crates/openshell-cli/src/lib.rs b/crates/openshell-cli/src/lib.rs index 09e05449b9..1746547ef6 100644 --- a/crates/openshell-cli/src/lib.rs +++ b/crates/openshell-cli/src/lib.rs @@ -12,6 +12,7 @@ pub mod auth; pub mod bootstrap; pub mod completers; pub mod edge_tunnel; +pub(crate) mod policy_update; pub mod run; pub mod ssh; pub mod tls; diff --git a/crates/openshell-cli/src/main.rs b/crates/openshell-cli/src/main.rs index 2929224118..8474afd15a 100644 --- a/crates/openshell-cli/src/main.rs +++ b/crates/openshell-cli/src/main.rs @@ -254,6 +254,8 @@ const POLICY_EXAMPLES: &str = "\x1b[1mALIAS\x1b[0m \x1b[1mEXAMPLES\x1b[0m $ openshell policy get my-sandbox $ openshell policy set my-sandbox --policy policy.yaml + $ openshell policy update my-sandbox --add-endpoint api.github.com:443:read-only:rest:enforce + $ openshell policy update my-sandbox --add-allow 'api.github.com:443:GET:/repos/**' $ openshell policy set --global --policy policy.yaml $ openshell policy delete --global $ openshell policy list my-sandbox @@ -1438,6 +1440,54 @@ enum PolicyCommands { timeout: u64, }, + /// Incrementally update policy on a live sandbox. + #[command(help_template = LEAF_HELP_TEMPLATE, next_help_heading = "FLAGS")] + Update { + /// Sandbox name (defaults to last-used sandbox). + #[arg(add = ArgValueCompleter::new(completers::complete_sandbox_names))] + name: Option, + + /// Add or merge an endpoint: host:port[:access[:protocol[:enforcement]]]. + #[arg(long = "add-endpoint")] + add_endpoints: Vec, + + /// Remove an endpoint: host:port. + #[arg(long = "remove-endpoint")] + remove_endpoints: Vec, + + /// Add a REST allow rule: host:port:METHOD:path_glob. + #[arg(long = "add-allow")] + add_allow: Vec, + + /// Add a REST deny rule: host:port:METHOD:path_glob. + #[arg(long = "add-deny")] + add_deny: Vec, + + /// Remove a network rule by name. + #[arg(long = "remove-rule")] + remove_rules: Vec, + + /// Add binaries to each --add-endpoint rule. + #[arg(long = "binary", value_hint = ValueHint::FilePath)] + binaries: Vec, + + /// Override the generated rule name when exactly one --add-endpoint is provided. + #[arg(long = "rule-name")] + rule_name: Option, + + /// Preview the merged policy without sending it to the gateway. + #[arg(long)] + dry_run: bool, + + /// Wait for the sandbox to load the policy revision. + #[arg(long)] + wait: bool, + + /// Timeout for --wait in seconds. + #[arg(long, default_value_t = 60)] + timeout: u64, + }, + /// Show current active policy for a sandbox or the global policy. #[command(help_template = LEAF_HELP_TEMPLATE, next_help_heading = "FLAGS")] Get { @@ -1988,6 +2038,37 @@ async fn main() -> Result<()> { .await?; } } + PolicyCommands::Update { + name, + add_endpoints, + remove_endpoints, + add_allow, + add_deny, + remove_rules, + binaries, + rule_name, + dry_run, + wait, + timeout, + } => { + let name = resolve_sandbox_name(name, &ctx.name)?; + run::sandbox_policy_update( + &ctx.endpoint, + &name, + &add_endpoints, + &remove_endpoints, + &add_deny, + &add_allow, + &remove_rules, + &binaries, + rule_name.as_deref(), + dry_run, + wait, + timeout, + &tls, + ) + .await?; + } PolicyCommands::Get { name, rev, diff --git a/crates/openshell-cli/src/policy_update.rs b/crates/openshell-cli/src/policy_update.rs new file mode 100644 index 0000000000..9f053f73b9 --- /dev/null +++ b/crates/openshell-cli/src/policy_update.rs @@ -0,0 +1,473 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::BTreeMap; + +use miette::{Result, miette}; +use openshell_core::proto::policy_merge_operation; +use openshell_core::proto::{ + AddAllowRules, AddDenyRules, AddNetworkRule, L7Allow, L7DenyRule, L7Rule, NetworkBinary, + NetworkEndpoint, NetworkPolicyRule, PolicyMergeOperation, RemoveNetworkEndpoint, + RemoveNetworkRule, +}; +use openshell_policy::{PolicyMergeOp, generated_rule_name}; + +#[derive(Debug, Clone)] +pub(crate) struct PolicyUpdatePlan { + pub merge_operations: Vec, + pub preview_operations: Vec, +} + +pub(crate) fn build_policy_update_plan( + add_endpoints: &[String], + remove_endpoints: &[String], + add_deny: &[String], + add_allow: &[String], + remove_rules: &[String], + binaries: &[String], + rule_name: Option<&str>, +) -> Result { + if binaries.iter().any(|binary| binary.trim().is_empty()) { + return Err(miette!("--binary values must not be empty")); + } + if !binaries.is_empty() && add_endpoints.is_empty() { + return Err(miette!("--binary can only be used with --add-endpoint")); + } + if rule_name.is_some() && add_endpoints.is_empty() { + return Err(miette!("--rule-name can only be used with --add-endpoint")); + } + if rule_name.is_some() && add_endpoints.len() > 1 { + return Err(miette!( + "--rule-name is only supported when exactly one --add-endpoint is provided" + )); + } + + let mut merge_operations = Vec::new(); + let mut preview_operations = Vec::new(); + + let deduped_binaries = dedup_strings(binaries); + for spec in add_endpoints { + let endpoint = parse_add_endpoint_spec(spec)?; + let target_rule_name = rule_name + .map(str::trim) + .filter(|name| !name.is_empty()) + .map(ToString::to_string) + .unwrap_or_else(|| generated_rule_name(&endpoint.host, endpoint.port)); + let rule = NetworkPolicyRule { + name: target_rule_name.clone(), + endpoints: vec![endpoint.clone()], + binaries: deduped_binaries + .iter() + .map(|path| NetworkBinary { + path: path.clone(), + ..Default::default() + }) + .collect(), + }; + merge_operations.push(PolicyMergeOperation { + operation: Some(policy_merge_operation::Operation::AddRule(AddNetworkRule { + rule_name: target_rule_name.clone(), + rule: Some(rule.clone()), + })), + }); + preview_operations.push(PolicyMergeOp::AddRule { + rule_name: target_rule_name, + rule, + }); + } + + for spec in remove_endpoints { + let (host, port) = parse_remove_endpoint_spec(spec)?; + merge_operations.push(PolicyMergeOperation { + operation: Some(policy_merge_operation::Operation::RemoveEndpoint( + RemoveNetworkEndpoint { + rule_name: String::new(), + host: host.clone(), + port, + }, + )), + }); + preview_operations.push(PolicyMergeOp::RemoveEndpoint { + rule_name: None, + host, + port, + }); + } + + for name in remove_rules { + let rule_name = name.trim(); + if rule_name.is_empty() { + return Err(miette!("--remove-rule values must not be empty")); + } + merge_operations.push(PolicyMergeOperation { + operation: Some(policy_merge_operation::Operation::RemoveRule( + RemoveNetworkRule { + rule_name: rule_name.to_string(), + }, + )), + }); + preview_operations.push(PolicyMergeOp::RemoveRule { + rule_name: rule_name.to_string(), + }); + } + + for ((host, port), rules) in group_allow_rules(add_allow)? { + merge_operations.push(PolicyMergeOperation { + operation: Some(policy_merge_operation::Operation::AddAllowRules( + AddAllowRules { + host: host.clone(), + port, + rules: rules.clone(), + }, + )), + }); + preview_operations.push(PolicyMergeOp::AddAllowRules { host, port, rules }); + } + + for ((host, port), deny_rules) in group_deny_rules(add_deny)? { + merge_operations.push(PolicyMergeOperation { + operation: Some(policy_merge_operation::Operation::AddDenyRules( + AddDenyRules { + host: host.clone(), + port, + deny_rules: deny_rules.clone(), + }, + )), + }); + preview_operations.push(PolicyMergeOp::AddDenyRules { + host, + port, + deny_rules, + }); + } + + if merge_operations.is_empty() { + return Err(miette!( + "policy update requires at least one operation flag" + )); + } + + Ok(PolicyUpdatePlan { + merge_operations, + preview_operations, + }) +} + +fn group_allow_rules(specs: &[String]) -> Result>> { + let mut grouped = BTreeMap::new(); + for spec in specs { + let parsed = parse_l7_rule_spec("--add-allow", spec)?; + grouped + .entry((parsed.host, parsed.port)) + .or_insert_with(Vec::new) + .push(L7Rule { + allow: Some(L7Allow { + method: parsed.method, + path: parsed.path, + command: String::new(), + query: Default::default(), + }), + }); + } + Ok(grouped) +} + +fn group_deny_rules(specs: &[String]) -> Result>> { + let mut grouped = BTreeMap::new(); + for spec in specs { + let parsed = parse_l7_rule_spec("--add-deny", spec)?; + grouped + .entry((parsed.host, parsed.port)) + .or_insert_with(Vec::new) + .push(L7DenyRule { + method: parsed.method, + path: parsed.path, + command: String::new(), + query: Default::default(), + }); + } + Ok(grouped) +} + +#[derive(Debug, Clone)] +struct ParsedL7RuleSpec { + host: String, + port: u32, + method: String, + path: String, +} + +fn parse_l7_rule_spec(flag: &str, spec: &str) -> Result { + let parts = spec.split(':').collect::>(); + if parts.len() != 4 { + return Err(miette!( + "{flag} expects host:port:METHOD:path_glob, got '{spec}'" + )); + } + + let host = parse_host(flag, spec, parts[0])?; + let port = parse_port(flag, spec, parts[1])?; + let method = parts[2].trim(); + if method.is_empty() { + return Err(miette!("{flag} has an empty METHOD segment in '{spec}'")); + } + if method.contains(char::is_whitespace) { + return Err(miette!( + "{flag} METHOD must not contain whitespace in '{spec}'" + )); + } + + let path = parts[3].trim(); + if path.is_empty() { + return Err(miette!("{flag} has an empty path segment in '{spec}'")); + } + if !path.starts_with('/') && path != "**" && !path.starts_with("**/") { + return Err(miette!( + "{flag} path must start with '/' or be '**', got '{path}' in '{spec}'" + )); + } + + Ok(ParsedL7RuleSpec { + host, + port, + method: method.to_ascii_uppercase(), + path: path.to_string(), + }) +} + +fn parse_remove_endpoint_spec(spec: &str) -> Result<(String, u32)> { + let parts = spec.split(':').collect::>(); + if parts.len() != 2 { + return Err(miette!("--remove-endpoint expects host:port, got '{spec}'")); + } + + Ok(( + parse_host("--remove-endpoint", spec, parts[0])?, + parse_port("--remove-endpoint", spec, parts[1])?, + )) +} + +fn parse_add_endpoint_spec(spec: &str) -> Result { + let parts = spec.split(':').collect::>(); + if !(2..=5).contains(&parts.len()) { + return Err(miette!( + "--add-endpoint expects host:port[:access[:protocol[:enforcement]]], got '{spec}'" + )); + } + + let host = parse_host("--add-endpoint", spec, parts[0])?; + let port = parse_port("--add-endpoint", spec, parts[1])?; + + let access = parts.get(2).copied().unwrap_or("").trim(); + let protocol = parts.get(3).copied().unwrap_or("").trim(); + let enforcement = parts.get(4).copied().unwrap_or("").trim(); + + if parts.len() == 3 && access.is_empty() { + return Err(miette!( + "--add-endpoint has an empty access segment in '{spec}'; omit it entirely if you do not need access or protocol fields" + )); + } + if !enforcement.is_empty() && protocol.is_empty() { + return Err(miette!( + "--add-endpoint cannot set enforcement without protocol in '{spec}'" + )); + } + if !access.is_empty() && !matches!(access, "read-only" | "read-write" | "full") { + return Err(miette!( + "--add-endpoint access segment must be one of read-only, read-write, or full; got '{access}' in '{spec}'" + )); + } + if !protocol.is_empty() && !matches!(protocol, "rest" | "sql") { + return Err(miette!( + "--add-endpoint protocol segment must be 'rest' or 'sql'; got '{protocol}' in '{spec}'" + )); + } + if !enforcement.is_empty() && !matches!(enforcement, "enforce" | "audit") { + return Err(miette!( + "--add-endpoint enforcement segment must be 'enforce' or 'audit'; got '{enforcement}' in '{spec}'" + )); + } + + Ok(NetworkEndpoint { + host, + port, + ports: vec![port], + protocol: protocol.to_string(), + enforcement: enforcement.to_string(), + access: access.to_string(), + ..Default::default() + }) +} + +fn parse_host(flag: &str, spec: &str, host: &str) -> Result { + let host = host.trim(); + if host.is_empty() { + return Err(miette!("{flag} has an empty host segment in '{spec}'")); + } + if host.contains(char::is_whitespace) { + return Err(miette!( + "{flag} host must not contain whitespace in '{spec}'" + )); + } + if host.contains('/') { + return Err(miette!("{flag} host must not contain '/' in '{spec}'")); + } + Ok(host.to_string()) +} + +fn parse_port(flag: &str, spec: &str, port: &str) -> Result { + let port = port.trim(); + if port.is_empty() { + return Err(miette!("{flag} has an empty port segment in '{spec}'")); + } + let parsed = port.parse::().map_err(|_| { + miette!("{flag} port segment must be a base-10 integer, got '{port}' in '{spec}'") + })?; + if parsed == 0 || parsed > 65535 { + return Err(miette!( + "{flag} port must be in the range 1-65535, got '{parsed}' in '{spec}'" + )); + } + Ok(parsed) +} + +fn dedup_strings(values: &[String]) -> Vec { + let mut deduped = Vec::new(); + for value in values { + let trimmed = value.trim(); + if !trimmed.is_empty() && !deduped.iter().any(|existing| existing == trimmed) { + deduped.push(trimmed.to_string()); + } + } + deduped +} + +#[cfg(test)] +mod tests { + use super::build_policy_update_plan; + + #[test] + fn parse_add_endpoint_basic_l4() { + let plan = + build_policy_update_plan(&["ghcr.io:443".to_string()], &[], &[], &[], &[], &[], None) + .expect("plan should build"); + assert_eq!(plan.merge_operations.len(), 1); + assert_eq!(plan.preview_operations.len(), 1); + } + + #[test] + fn parse_add_endpoint_rejects_bad_access() { + let error = build_policy_update_plan( + &["api.github.com:443:write-ish".to_string()], + &[], + &[], + &[], + &[], + &[], + None, + ) + .expect_err("plan should fail"); + assert!(error.to_string().contains("access segment")); + } + + #[test] + fn parse_add_endpoint_allows_empty_access_when_protocol_present() { + build_policy_update_plan( + &["api.github.com:443::rest:enforce".to_string()], + &[], + &[], + &[], + &[], + &[], + None, + ) + .expect("plan should build"); + } + + #[test] + fn parse_add_deny_rejects_empty_method() { + let error = build_policy_update_plan( + &[], + &[], + &["api.github.com:443::/repos/**".to_string()], + &[], + &[], + &[], + None, + ) + .expect_err("plan should fail"); + assert!(error.to_string().contains("METHOD")); + } + + #[test] + fn parse_add_allow_rejects_non_absolute_path() { + let error = build_policy_update_plan( + &[], + &[], + &[], + &["api.github.com:443:GET:repos/**".to_string()], + &[], + &[], + None, + ) + .expect_err("plan should fail"); + assert!(error.to_string().contains("path must start with '/'")); + } + + #[test] + fn parse_add_endpoint_rejects_enforcement_without_protocol() { + let error = build_policy_update_plan( + &["api.github.com:443:read-only::enforce".to_string()], + &[], + &[], + &[], + &[], + &[], + None, + ) + .expect_err("plan should fail"); + assert!( + error + .to_string() + .contains("cannot set enforcement without protocol") + ); + } + + #[test] + fn parse_remove_endpoint_rejects_out_of_range_port() { + let error = build_policy_update_plan( + &[], + &["api.github.com:70000".to_string()], + &[], + &[], + &[], + &[], + None, + ) + .expect_err("plan should fail"); + assert!(error.to_string().contains("range 1-65535")); + } + + #[test] + fn binary_requires_add_endpoint() { + let error = + build_policy_update_plan(&[], &[], &[], &[], &[], &["/usr/bin/gh".to_string()], None) + .expect_err("plan should fail"); + assert!(error.to_string().contains("--binary")); + } + + #[test] + fn rule_name_rejects_multiple_add_endpoints() { + let error = build_policy_update_plan( + &["api.github.com:443".to_string(), "ghcr.io:443".to_string()], + &[], + &[], + &[], + &[], + &[], + Some("shared"), + ) + .expect_err("plan should fail"); + assert!(error.to_string().contains("exactly one --add-endpoint")); + } +} diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index c41b53518e..60e28ac7e1 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -3,6 +3,7 @@ //! CLI command implementations. +use crate::policy_update::build_policy_update_plan; use crate::tls::{ TlsOptions, build_rustls_config, grpc_client, grpc_inference_client, require_tls_materials, }; @@ -27,7 +28,7 @@ use openshell_core::proto::{ ExecSandboxRequest, GetClusterInferenceRequest, GetDraftHistoryRequest, GetDraftPolicyRequest, GetGatewayConfigRequest, GetProviderRequest, GetSandboxConfigRequest, GetSandboxLogsRequest, GetSandboxPolicyStatusRequest, GetSandboxRequest, HealthRequest, ListProvidersRequest, - ListSandboxPoliciesRequest, ListSandboxesRequest, PolicyStatus, Provider, + ListSandboxPoliciesRequest, ListSandboxesRequest, PolicySource, PolicyStatus, Provider, RejectDraftChunkRequest, Sandbox, SandboxPhase, SandboxPolicy, SandboxSpec, SandboxTemplate, SetClusterInferenceRequest, SettingScope, SettingValue, UpdateConfigRequest, UpdateProviderRequest, WatchSandboxRequest, exec_sandbox_event, setting_value, @@ -4144,6 +4145,16 @@ fn format_setting_value(value: Option<&SettingValue>) -> String { } } +fn short_hash(hash: &str) -> &str { + if hash.len() >= 12 { &hash[..12] } else { hash } +} + +fn print_policy_merge_warnings(warnings: &[openshell_policy::PolicyMergeWarning]) { + for warning in warnings { + eprintln!("{} {}", "!".yellow().bold(), warning); + } +} + pub async fn sandbox_policy_set_global( server: &str, policy_path: &str, @@ -4172,6 +4183,7 @@ pub async fn sandbox_policy_set_global( setting_value: None, delete_setting: false, global: true, + merge_operations: vec![], }) .await .into_diagnostic()? @@ -4221,12 +4233,11 @@ pub async fn sandbox_settings_get( return Ok(()); } - let policy_source = - if response.policy_source == openshell_core::proto::PolicySource::Global as i32 { - "global" - } else { - "sandbox" - }; + let policy_source = if response.policy_source == PolicySource::Global as i32 { + "global" + } else { + "sandbox" + }; println!("Sandbox: {}", name); println!("Config Rev: {}", response.config_revision); @@ -4297,12 +4308,11 @@ fn settings_to_json_sandbox( name: &str, response: &openshell_core::proto::GetSandboxConfigResponse, ) -> serde_json::Value { - let policy_source = - if response.policy_source == openshell_core::proto::PolicySource::Global as i32 { - "global" - } else { - "sandbox" - }; + let policy_source = if response.policy_source == PolicySource::Global as i32 { + "global" + } else { + "sandbox" + }; let mut settings = serde_json::Map::new(); let mut keys: Vec<_> = response.settings.keys().cloned().collect(); @@ -4371,6 +4381,7 @@ pub async fn gateway_setting_set( setting_value: Some(setting_value), delete_setting: false, global: true, + merge_operations: vec![], }) .await .into_diagnostic()? @@ -4404,6 +4415,7 @@ pub async fn sandbox_setting_set( setting_value: Some(setting_value), delete_setting: false, global: false, + merge_operations: vec![], }) .await .into_diagnostic()? @@ -4437,6 +4449,7 @@ pub async fn gateway_setting_delete( setting_value: None, delete_setting: true, global: true, + merge_operations: vec![], }) .await .into_diagnostic()? @@ -4470,6 +4483,7 @@ pub async fn sandbox_setting_delete( setting_value: None, delete_setting: true, global: false, + merge_operations: vec![], }) .await .into_diagnostic()? @@ -4527,6 +4541,7 @@ pub async fn sandbox_policy_set( setting_value: None, delete_setting: false, global: false, + merge_operations: vec![], }) .await .into_diagnostic()?; @@ -4614,6 +4629,176 @@ pub async fn sandbox_policy_set( } } +#[allow(clippy::too_many_arguments)] +pub async fn sandbox_policy_update( + server: &str, + name: &str, + add_endpoints: &[String], + remove_endpoints: &[String], + add_deny: &[String], + add_allow: &[String], + remove_rules: &[String], + binaries: &[String], + rule_name: Option<&str>, + dry_run: bool, + wait: bool, + timeout_secs: u64, + tls: &TlsOptions, +) -> Result<()> { + if dry_run && wait { + return Err(miette!("--wait cannot be combined with --dry-run")); + } + + let plan = build_policy_update_plan( + add_endpoints, + remove_endpoints, + add_deny, + add_allow, + remove_rules, + binaries, + rule_name, + )?; + + let mut client = grpc_client(server, tls).await?; + let sandbox = client + .get_sandbox(GetSandboxRequest { + name: name.to_string(), + }) + .await + .into_diagnostic()? + .into_inner() + .sandbox + .ok_or_else(|| miette!("sandbox not found"))?; + + let current = client + .get_sandbox_config(GetSandboxConfigRequest { + sandbox_id: sandbox.id.clone(), + }) + .await + .into_diagnostic()? + .into_inner(); + + if current.policy_source == PolicySource::Global as i32 { + return Err(miette!( + "policy is managed globally; delete the global policy before using `openshell policy update`" + )); + } + + let merged = openshell_policy::merge_policy( + current.policy.clone().unwrap_or_default(), + &plan.preview_operations, + ) + .map_err(|error| miette!("{error}"))?; + + if dry_run { + eprintln!( + "{} Dry run preview for {} incremental policy operation(s)", + "✓".green().bold(), + plan.preview_operations.len() + ); + print_policy_merge_warnings(&merged.warnings); + print_sandbox_policy(&merged.policy); + return Ok(()); + } + + let current_version = current.version; + let current_hash = current.policy_hash.clone(); + let response = client + .update_config(UpdateConfigRequest { + name: name.to_string(), + policy: None, + setting_key: String::new(), + setting_value: None, + delete_setting: false, + global: false, + merge_operations: plan.merge_operations, + }) + .await + .into_diagnostic()? + .into_inner(); + + print_policy_merge_warnings(&merged.warnings); + + if response.version == current_version && response.policy_hash == current_hash { + eprintln!( + "{} Policy unchanged (version {}, hash: {})", + "·".dimmed(), + response.version, + short_hash(&response.policy_hash) + ); + return Ok(()); + } + + eprintln!( + "{} Policy version {} submitted (hash: {})", + "✓".green().bold(), + response.version, + short_hash(&response.policy_hash) + ); + + if !wait { + return Ok(()); + } + + let deadline = Instant::now() + Duration::from_secs(timeout_secs); + loop { + if Instant::now() > deadline { + eprintln!( + "{} Timeout waiting for policy version {} to load", + "✗".red().bold(), + response.version + ); + std::process::exit(124); + } + + tokio::time::sleep(Duration::from_secs(1)).await; + + let status_resp = client + .get_sandbox_policy_status(GetSandboxPolicyStatusRequest { + name: name.to_string(), + version: response.version, + global: false, + }) + .await + .into_diagnostic()?; + + let inner = status_resp.into_inner(); + if let Some(rev) = &inner.revision { + let status = PolicyStatus::try_from(rev.status).unwrap_or(PolicyStatus::Unspecified); + match status { + PolicyStatus::Loaded => { + eprintln!( + "{} Policy version {} loaded (active version: {})", + "✓".green().bold(), + rev.version, + inner.active_version + ); + return Ok(()); + } + PolicyStatus::Failed => { + eprintln!( + "{} Policy version {} failed to load: {}", + "✗".red().bold(), + rev.version, + rev.load_error + ); + std::process::exit(1); + } + PolicyStatus::Superseded => { + eprintln!( + "{} Policy version {} was superseded (active version: {})", + "⚠".yellow().bold(), + rev.version, + inner.active_version + ); + return Ok(()); + } + _ => {} + } + } + } +} + pub async fn sandbox_policy_get( server: &str, name: &str, diff --git a/crates/openshell-policy/src/lib.rs b/crates/openshell-policy/src/lib.rs index e3c26061a1..2c8c0cc769 100644 --- a/crates/openshell-policy/src/lib.rs +++ b/crates/openshell-policy/src/lib.rs @@ -9,6 +9,8 @@ //! policy schema. Both parsing (YAML→proto) and serialization (proto→YAML) use //! these types, ensuring round-trip fidelity. +mod merge; + use std::collections::{BTreeMap, HashMap}; use std::fmt; use std::path::Path; @@ -20,6 +22,11 @@ use openshell_core::proto::{ }; use serde::{Deserialize, Serialize}; +pub use merge::{ + PolicyMergeError, PolicyMergeOp, PolicyMergeResult, PolicyMergeWarning, generated_rule_name, + merge_policy, +}; + // --------------------------------------------------------------------------- // YAML serde types (canonical — used for both parsing and serialization) // --------------------------------------------------------------------------- diff --git a/crates/openshell-policy/src/merge.rs b/crates/openshell-policy/src/merge.rs new file mode 100644 index 0000000000..5f5d2d40dd --- /dev/null +++ b/crates/openshell-policy/src/merge.rs @@ -0,0 +1,1016 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::HashSet; + +use openshell_core::proto::{ + L7Allow, L7DenyRule, L7Rule, NetworkBinary, NetworkEndpoint, NetworkPolicyRule, SandboxPolicy, +}; + +#[derive(Debug, Clone, PartialEq)] +pub enum PolicyMergeOp { + AddRule { + rule_name: String, + rule: NetworkPolicyRule, + }, + RemoveEndpoint { + rule_name: Option, + host: String, + port: u32, + }, + RemoveRule { + rule_name: String, + }, + AddDenyRules { + host: String, + port: u32, + deny_rules: Vec, + }, + AddAllowRules { + host: String, + port: u32, + rules: Vec, + }, + RemoveBinary { + rule_name: String, + binary_path: String, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PolicyMergeWarning { + ExistingProtocolRetained { + host: String, + port: u32, + existing: String, + incoming: String, + }, + ExistingEnforcementRetained { + host: String, + port: u32, + existing: String, + incoming: String, + }, + ExistingTlsRetained { + host: String, + port: u32, + existing: String, + incoming: String, + }, + ExistingAccessRetained { + host: String, + port: u32, + existing: String, + incoming: String, + }, + ExpandedAccessPreset { + host: String, + port: u32, + access: String, + }, + IgnoredIncomingAccessBecauseRulesExist { + host: String, + port: u32, + incoming: String, + }, +} + +impl std::fmt::Display for PolicyMergeWarning { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::ExistingProtocolRetained { + host, + port, + existing, + incoming, + } => write!( + f, + "endpoint {host}:{port} keeps existing protocol '{existing}' and ignores incoming '{incoming}'" + ), + Self::ExistingEnforcementRetained { + host, + port, + existing, + incoming, + } => write!( + f, + "endpoint {host}:{port} keeps existing enforcement '{existing}' and ignores incoming '{incoming}'" + ), + Self::ExistingTlsRetained { + host, + port, + existing, + incoming, + } => write!( + f, + "endpoint {host}:{port} keeps existing tls mode '{existing}' and ignores incoming '{incoming}'" + ), + Self::ExistingAccessRetained { + host, + port, + existing, + incoming, + } => write!( + f, + "endpoint {host}:{port} keeps existing access preset '{existing}' and ignores incoming '{incoming}'" + ), + Self::ExpandedAccessPreset { host, port, access } => write!( + f, + "expanded access preset '{access}' to explicit rules for endpoint {host}:{port}" + ), + Self::IgnoredIncomingAccessBecauseRulesExist { + host, + port, + incoming, + } => write!( + f, + "endpoint {host}:{port} already uses explicit rules; incoming access preset '{incoming}' was ignored" + ), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PolicyMergeError { + MissingRuleNameForAddRule, + InvalidEndpointReference { + host: String, + port: u32, + }, + EndpointNotFound { + host: String, + port: u32, + }, + EndpointHasNoL7Inspection { + host: String, + port: u32, + }, + UnsupportedEndpointProtocol { + host: String, + port: u32, + protocol: String, + }, + EndpointHasNoAllowBase { + host: String, + port: u32, + }, + UnsupportedAccessPreset { + host: String, + port: u32, + access: String, + }, +} + +impl std::fmt::Display for PolicyMergeError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::MissingRuleNameForAddRule => write!(f, "add-rule operation requires a rule name"), + Self::InvalidEndpointReference { host, port } => { + write!(f, "invalid endpoint reference '{host}:{port}'") + } + Self::EndpointNotFound { host, port } => { + write!( + f, + "endpoint {host}:{port} was not found in the current policy" + ) + } + Self::EndpointHasNoL7Inspection { host, port } => write!( + f, + "endpoint {host}:{port} has no L7 inspection configured (protocol is empty)" + ), + Self::UnsupportedEndpointProtocol { + host, + port, + protocol, + } => write!( + f, + "endpoint {host}:{port} uses unsupported protocol '{protocol}'; this operation currently supports only protocol 'rest'" + ), + Self::EndpointHasNoAllowBase { host, port } => write!( + f, + "endpoint {host}:{port} has no base allow set; configure access or explicit allow rules before adding deny rules" + ), + Self::UnsupportedAccessPreset { host, port, access } => write!( + f, + "endpoint {host}:{port} uses unsupported access preset '{access}'" + ), + } + } +} + +impl std::error::Error for PolicyMergeError {} + +#[derive(Debug, Clone, PartialEq)] +pub struct PolicyMergeResult { + pub policy: SandboxPolicy, + pub warnings: Vec, + pub changed: bool, +} + +pub fn merge_policy( + policy: SandboxPolicy, + operations: &[PolicyMergeOp], +) -> Result { + let mut merged = policy.clone(); + let mut warnings = Vec::new(); + + for operation in operations { + apply_operation(&mut merged, operation, &mut warnings)?; + } + + let changed = merged != policy; + Ok(PolicyMergeResult { + policy: merged, + warnings, + changed, + }) +} + +pub fn generated_rule_name(host: &str, port: u32) -> String { + let sanitized = host + .replace(['.', '-'], "_") + .chars() + .filter(|c| c.is_alphanumeric() || *c == '_') + .collect::(); + format!("allow_{sanitized}_{port}") +} + +fn apply_operation( + policy: &mut SandboxPolicy, + operation: &PolicyMergeOp, + warnings: &mut Vec, +) -> Result<(), PolicyMergeError> { + match operation { + PolicyMergeOp::AddRule { rule_name, rule } => { + add_rule(policy, rule_name, rule, warnings)?; + } + PolicyMergeOp::RemoveEndpoint { + rule_name, + host, + port, + } => { + remove_endpoint(policy, rule_name.as_deref(), host, *port); + } + PolicyMergeOp::RemoveRule { rule_name } => { + policy.network_policies.remove(rule_name); + } + PolicyMergeOp::AddDenyRules { + host, + port, + deny_rules, + } => { + let endpoint = find_endpoint_mut(policy, host, *port).ok_or_else(|| { + PolicyMergeError::EndpointNotFound { + host: host.clone(), + port: *port, + } + })?; + ensure_rest_endpoint(endpoint, host, *port)?; + if endpoint.access.is_empty() && endpoint.rules.is_empty() { + return Err(PolicyMergeError::EndpointHasNoAllowBase { + host: host.clone(), + port: *port, + }); + } + append_unique_deny_rules(&mut endpoint.deny_rules, deny_rules); + } + PolicyMergeOp::AddAllowRules { host, port, rules } => { + let endpoint = find_endpoint_mut(policy, host, *port).ok_or_else(|| { + PolicyMergeError::EndpointNotFound { + host: host.clone(), + port: *port, + } + })?; + ensure_rest_endpoint(endpoint, host, *port)?; + expand_existing_access(endpoint, host, *port, warnings)?; + append_unique_l7_rules(&mut endpoint.rules, rules); + } + PolicyMergeOp::RemoveBinary { + rule_name, + binary_path, + } => { + let should_remove = if let Some(rule) = policy.network_policies.get_mut(rule_name) { + let original_len = rule.binaries.len(); + rule.binaries.retain(|binary| binary.path != *binary_path); + original_len != rule.binaries.len() && rule.binaries.is_empty() + } else { + false + }; + if should_remove { + policy.network_policies.remove(rule_name); + } + } + } + Ok(()) +} + +fn add_rule( + policy: &mut SandboxPolicy, + rule_name: &str, + incoming_rule: &NetworkPolicyRule, + warnings: &mut Vec, +) -> Result<(), PolicyMergeError> { + if rule_name.trim().is_empty() { + return Err(PolicyMergeError::MissingRuleNameForAddRule); + } + + let mut incoming_rule = incoming_rule.clone(); + normalize_rule(&mut incoming_rule); + if incoming_rule.name.is_empty() { + incoming_rule.name = rule_name.to_string(); + } + + let target_key = if policy.network_policies.contains_key(rule_name) { + Some(rule_name.to_string()) + } else { + let mut keys: Vec<_> = policy.network_policies.keys().cloned().collect(); + keys.sort(); + keys.into_iter().find(|key| { + policy + .network_policies + .get(key) + .is_some_and(|existing_rule| rules_share_endpoint(existing_rule, &incoming_rule)) + }) + }; + + if let Some(key) = target_key { + let existing_rule = policy + .network_policies + .get_mut(&key) + .expect("existing rule must be present"); + merge_rules(existing_rule, &incoming_rule, warnings)?; + } else { + policy + .network_policies + .insert(rule_name.to_string(), incoming_rule); + } + + Ok(()) +} + +fn merge_rules( + existing_rule: &mut NetworkPolicyRule, + incoming_rule: &NetworkPolicyRule, + warnings: &mut Vec, +) -> Result<(), PolicyMergeError> { + append_unique_binaries(&mut existing_rule.binaries, &incoming_rule.binaries); + + for incoming_endpoint in &incoming_rule.endpoints { + let mut incoming_endpoint = incoming_endpoint.clone(); + normalize_endpoint(&mut incoming_endpoint); + if let Some(existing_endpoint) = + find_matching_endpoint_mut(&mut existing_rule.endpoints, &incoming_endpoint) + { + merge_endpoint(existing_endpoint, &incoming_endpoint, warnings)?; + } else { + existing_rule.endpoints.push(incoming_endpoint); + } + } + + Ok(()) +} + +fn merge_endpoint( + existing: &mut NetworkEndpoint, + incoming: &NetworkEndpoint, + warnings: &mut Vec, +) -> Result<(), PolicyMergeError> { + let host = if existing.host.is_empty() { + incoming.host.clone() + } else { + existing.host.clone() + }; + let port = canonical_ports(existing) + .into_iter() + .next() + .or_else(|| canonical_ports(incoming).into_iter().next()) + .unwrap_or(0); + + if existing.host.is_empty() { + existing.host = incoming.host.clone(); + } + + merge_endpoint_ports(existing, incoming); + let existing_protocol = existing.protocol.clone(); + merge_string_field( + &mut existing.protocol, + &incoming.protocol, + PolicyMergeWarning::ExistingProtocolRetained { + host: host.clone(), + port, + existing: existing_protocol, + incoming: incoming.protocol.clone(), + }, + warnings, + ); + let existing_enforcement = existing.enforcement.clone(); + merge_string_field( + &mut existing.enforcement, + &incoming.enforcement, + PolicyMergeWarning::ExistingEnforcementRetained { + host: host.clone(), + port, + existing: existing_enforcement, + incoming: incoming.enforcement.clone(), + }, + warnings, + ); + let existing_tls = existing.tls.clone(); + merge_string_field( + &mut existing.tls, + &incoming.tls, + PolicyMergeWarning::ExistingTlsRetained { + host: host.clone(), + port, + existing: existing_tls, + incoming: incoming.tls.clone(), + }, + warnings, + ); + + if !incoming.rules.is_empty() { + expand_existing_access(existing, &host, port, warnings)?; + append_unique_l7_rules(&mut existing.rules, &incoming.rules); + if !incoming.access.is_empty() { + warnings.push(PolicyMergeWarning::IgnoredIncomingAccessBecauseRulesExist { + host, + port, + incoming: incoming.access.clone(), + }); + } + } else if !incoming.access.is_empty() { + if !existing.rules.is_empty() { + warnings.push(PolicyMergeWarning::IgnoredIncomingAccessBecauseRulesExist { + host, + port, + incoming: incoming.access.clone(), + }); + } else if existing.access.is_empty() { + existing.access = incoming.access.clone(); + } else if existing.access != incoming.access { + warnings.push(PolicyMergeWarning::ExistingAccessRetained { + host, + port, + existing: existing.access.clone(), + incoming: incoming.access.clone(), + }); + } + } + + append_unique_deny_rules(&mut existing.deny_rules, &incoming.deny_rules); + append_unique_strings(&mut existing.allowed_ips, &incoming.allowed_ips); + normalize_endpoint(existing); + Ok(()) +} + +fn merge_string_field( + existing: &mut String, + incoming: &str, + warning: PolicyMergeWarning, + warnings: &mut Vec, +) { + if incoming.is_empty() { + return; + } + if existing.is_empty() { + *existing = incoming.to_string(); + } else if *existing != incoming { + warnings.push(warning); + } +} + +fn merge_endpoint_ports(existing: &mut NetworkEndpoint, incoming: &NetworkEndpoint) { + let mut ports = canonical_ports(existing); + for port in canonical_ports(incoming) { + if !ports.contains(&port) { + ports.push(port); + } + } + ports.sort_unstable(); + ports.dedup(); + existing.ports = ports.clone(); + existing.port = ports.first().copied().unwrap_or(0); +} + +fn rules_share_endpoint( + existing_rule: &NetworkPolicyRule, + incoming_rule: &NetworkPolicyRule, +) -> bool { + incoming_rule.endpoints.iter().any(|incoming_endpoint| { + existing_rule + .endpoints + .iter() + .any(|existing_endpoint| endpoints_overlap(existing_endpoint, incoming_endpoint)) + }) +} + +fn endpoints_overlap(left: &NetworkEndpoint, right: &NetworkEndpoint) -> bool { + if !left.host.eq_ignore_ascii_case(&right.host) { + return false; + } + + let left_ports = canonical_ports(left); + let right_ports = canonical_ports(right); + left_ports.iter().any(|port| right_ports.contains(port)) +} + +fn canonical_ports(endpoint: &NetworkEndpoint) -> Vec { + if !endpoint.ports.is_empty() { + endpoint.ports.clone() + } else if endpoint.port > 0 { + vec![endpoint.port] + } else { + vec![] + } +} + +fn find_matching_endpoint_mut<'a>( + endpoints: &'a mut [NetworkEndpoint], + target: &NetworkEndpoint, +) -> Option<&'a mut NetworkEndpoint> { + endpoints + .iter_mut() + .find(|endpoint| endpoints_overlap(endpoint, target)) +} + +fn find_endpoint_mut<'a>( + policy: &'a mut SandboxPolicy, + host: &str, + port: u32, +) -> Option<&'a mut NetworkEndpoint> { + let mut keys: Vec<_> = policy.network_policies.keys().cloned().collect(); + keys.sort(); + let target_key = keys.into_iter().find(|key| { + policy.network_policies.get(key).is_some_and(|rule| { + rule.endpoints + .iter() + .any(|endpoint| endpoint_matches_host_port(endpoint, host, port)) + }) + })?; + + policy + .network_policies + .get_mut(&target_key) + .and_then(|rule| { + rule.endpoints + .iter_mut() + .find(|endpoint| endpoint_matches_host_port(endpoint, host, port)) + }) +} + +fn endpoint_matches_host_port(endpoint: &NetworkEndpoint, host: &str, port: u32) -> bool { + endpoint.host.eq_ignore_ascii_case(host) && canonical_ports(endpoint).contains(&port) +} + +fn ensure_rest_endpoint( + endpoint: &NetworkEndpoint, + host: &str, + port: u32, +) -> Result<(), PolicyMergeError> { + if endpoint.protocol.is_empty() { + return Err(PolicyMergeError::EndpointHasNoL7Inspection { + host: host.to_string(), + port, + }); + } + if endpoint.protocol != "rest" { + return Err(PolicyMergeError::UnsupportedEndpointProtocol { + host: host.to_string(), + port, + protocol: endpoint.protocol.clone(), + }); + } + Ok(()) +} + +fn expand_existing_access( + endpoint: &mut NetworkEndpoint, + host: &str, + port: u32, + warnings: &mut Vec, +) -> Result<(), PolicyMergeError> { + if endpoint.access.is_empty() { + return Ok(()); + } + + let access = endpoint.access.clone(); + let expanded = + expand_access_preset(&access).ok_or_else(|| PolicyMergeError::UnsupportedAccessPreset { + host: host.to_string(), + port, + access: access.clone(), + })?; + endpoint.access.clear(); + append_unique_l7_rules(&mut endpoint.rules, &expanded); + warnings.push(PolicyMergeWarning::ExpandedAccessPreset { + host: host.to_string(), + port, + access, + }); + Ok(()) +} + +fn expand_access_preset(access: &str) -> Option> { + let methods = match access { + "read-only" => vec!["GET", "HEAD", "OPTIONS"], + "read-write" => vec!["GET", "HEAD", "OPTIONS", "POST", "PUT", "PATCH"], + "full" => vec!["*"], + _ => return None, + }; + + Some( + methods + .into_iter() + .map(|method| L7Rule { + allow: Some(L7Allow { + method: method.to_string(), + path: "**".to_string(), + command: String::new(), + query: Default::default(), + }), + }) + .collect(), + ) +} + +fn append_unique_binaries(existing: &mut Vec, incoming: &[NetworkBinary]) { + let mut seen: HashSet = existing.iter().map(|binary| binary.path.clone()).collect(); + for binary in incoming { + if seen.insert(binary.path.clone()) { + existing.push(binary.clone()); + } + } +} + +fn append_unique_strings(existing: &mut Vec, incoming: &[String]) { + let mut seen: HashSet = existing.iter().cloned().collect(); + for value in incoming { + if seen.insert(value.clone()) { + existing.push(value.clone()); + } + } +} + +fn append_unique_l7_rules(existing: &mut Vec, incoming: &[L7Rule]) { + for rule in incoming { + if !existing.contains(rule) { + existing.push(rule.clone()); + } + } +} + +fn append_unique_deny_rules(existing: &mut Vec, incoming: &[L7DenyRule]) { + for rule in incoming { + if !existing.contains(rule) { + existing.push(rule.clone()); + } + } +} + +fn normalize_rule(rule: &mut NetworkPolicyRule) { + for endpoint in &mut rule.endpoints { + normalize_endpoint(endpoint); + } + dedup_binaries(&mut rule.binaries); +} + +fn normalize_endpoint(endpoint: &mut NetworkEndpoint) { + let mut ports = canonical_ports(endpoint); + ports.sort_unstable(); + ports.dedup(); + endpoint.ports = ports.clone(); + endpoint.port = ports.first().copied().unwrap_or(0); + dedup_strings(&mut endpoint.allowed_ips); + dedup_l7_rules(&mut endpoint.rules); + dedup_deny_rules(&mut endpoint.deny_rules); +} + +fn dedup_strings(values: &mut Vec) { + let mut seen = HashSet::new(); + values.retain(|value| seen.insert(value.clone())); +} + +fn dedup_binaries(values: &mut Vec) { + let mut seen = HashSet::new(); + values.retain(|binary| seen.insert(binary.path.clone())); +} + +fn dedup_l7_rules(values: &mut Vec) { + let mut deduped = Vec::with_capacity(values.len()); + for value in std::mem::take(values) { + if !deduped.contains(&value) { + deduped.push(value); + } + } + *values = deduped; +} + +fn dedup_deny_rules(values: &mut Vec) { + let mut deduped = Vec::with_capacity(values.len()); + for value in std::mem::take(values) { + if !deduped.contains(&value) { + deduped.push(value); + } + } + *values = deduped; +} + +fn remove_endpoint(policy: &mut SandboxPolicy, rule_name: Option<&str>, host: &str, port: u32) { + let target_keys: Vec = if let Some(rule_name) = rule_name { + if policy.network_policies.contains_key(rule_name) { + vec![rule_name.to_string()] + } else { + vec![] + } + } else { + let mut keys: Vec<_> = policy.network_policies.keys().cloned().collect(); + keys.sort(); + keys + }; + + let mut empty_rules = Vec::new(); + for key in target_keys { + if let Some(rule) = policy.network_policies.get_mut(&key) { + rule.endpoints.retain_mut(|endpoint| { + if !endpoint_matches_host_port(endpoint, host, port) { + return true; + } + + let mut remaining_ports = canonical_ports(endpoint); + remaining_ports.retain(|existing_port| *existing_port != port); + remaining_ports.sort_unstable(); + remaining_ports.dedup(); + + if remaining_ports.is_empty() { + return false; + } + + endpoint.ports = remaining_ports.clone(); + endpoint.port = remaining_ports[0]; + true + }); + + if rule.endpoints.is_empty() { + empty_rules.push(key); + } + } + } + + for key in empty_rules { + policy.network_policies.remove(&key); + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::{ + PolicyMergeError, PolicyMergeOp, PolicyMergeWarning, generated_rule_name, merge_policy, + }; + use crate::restrictive_default_policy; + use openshell_core::proto::{ + L7Allow, L7DenyRule, L7Rule, NetworkBinary, NetworkEndpoint, NetworkPolicyRule, + }; + + fn endpoint(host: &str, port: u32) -> NetworkEndpoint { + NetworkEndpoint { + host: host.to_string(), + port, + ports: vec![port], + ..Default::default() + } + } + + fn rule_with_endpoint(name: &str, host: &str, port: u32) -> NetworkPolicyRule { + NetworkPolicyRule { + name: name.to_string(), + endpoints: vec![endpoint(host, port)], + ..Default::default() + } + } + + fn rest_rule(method: &str, path: &str) -> L7Rule { + L7Rule { + allow: Some(L7Allow { + method: method.to_string(), + path: path.to_string(), + command: String::new(), + query: HashMap::new(), + }), + } + } + + #[test] + fn generated_rule_name_sanitizes_host() { + assert_eq!( + generated_rule_name("api.github.com", 443), + "allow_api_github_com_443" + ); + } + + #[test] + fn add_rule_merges_l7_fields_into_existing_endpoint() { + let mut policy = restrictive_default_policy(); + policy.network_policies.insert( + "existing".to_string(), + NetworkPolicyRule { + name: "existing".to_string(), + endpoints: vec![endpoint("api.github.com", 443)], + binaries: vec![NetworkBinary { + path: "/usr/bin/curl".to_string(), + ..Default::default() + }], + }, + ); + + let incoming = NetworkPolicyRule { + name: "incoming".to_string(), + endpoints: vec![NetworkEndpoint { + host: "api.github.com".to_string(), + port: 443, + ports: vec![443], + protocol: "rest".to_string(), + enforcement: "enforce".to_string(), + rules: vec![rest_rule("GET", "/repos/**")], + ..Default::default() + }], + binaries: vec![NetworkBinary { + path: "/usr/bin/gh".to_string(), + ..Default::default() + }], + }; + + let result = merge_policy( + policy, + &[PolicyMergeOp::AddRule { + rule_name: "allow_api_github_com_443".to_string(), + rule: incoming, + }], + ) + .expect("merge should succeed"); + + let rule = &result.policy.network_policies["existing"]; + let endpoint = &rule.endpoints[0]; + assert_eq!(endpoint.protocol, "rest"); + assert_eq!(endpoint.enforcement, "enforce"); + assert_eq!(endpoint.rules.len(), 1); + assert_eq!(rule.binaries.len(), 2); + } + + #[test] + fn add_allow_expands_access_preset() { + let mut policy = restrictive_default_policy(); + policy.network_policies.insert( + "github".to_string(), + NetworkPolicyRule { + name: "github".to_string(), + endpoints: vec![NetworkEndpoint { + host: "api.github.com".to_string(), + port: 443, + ports: vec![443], + protocol: "rest".to_string(), + access: "read-only".to_string(), + ..Default::default() + }], + ..Default::default() + }, + ); + + let result = merge_policy( + policy, + &[PolicyMergeOp::AddAllowRules { + host: "api.github.com".to_string(), + port: 443, + rules: vec![rest_rule("POST", "/repos/*/issues")], + }], + ) + .expect("merge should succeed"); + + let endpoint = &result.policy.network_policies["github"].endpoints[0]; + assert!(endpoint.access.is_empty()); + assert_eq!(endpoint.rules.len(), 4); + assert!(result.warnings.iter().any(|warning| matches!( + warning, + PolicyMergeWarning::ExpandedAccessPreset { access, .. } if access == "read-only" + ))); + } + + #[test] + fn add_deny_requires_rest_protocol() { + let mut policy = restrictive_default_policy(); + policy.network_policies.insert( + "db".to_string(), + NetworkPolicyRule { + name: "db".to_string(), + endpoints: vec![NetworkEndpoint { + host: "db.example.com".to_string(), + port: 5432, + ports: vec![5432], + protocol: "sql".to_string(), + access: "full".to_string(), + ..Default::default() + }], + ..Default::default() + }, + ); + + let error = merge_policy( + policy, + &[PolicyMergeOp::AddDenyRules { + host: "db.example.com".to_string(), + port: 5432, + deny_rules: vec![L7DenyRule { + method: "POST".to_string(), + path: "/admin".to_string(), + ..Default::default() + }], + }], + ) + .expect_err("merge should fail"); + + assert!(matches!( + error, + PolicyMergeError::UnsupportedEndpointProtocol { protocol, .. } if protocol == "sql" + )); + } + + #[test] + fn remove_endpoint_drops_only_requested_port() { + let mut policy = restrictive_default_policy(); + policy.network_policies.insert( + "multi".to_string(), + NetworkPolicyRule { + name: "multi".to_string(), + endpoints: vec![NetworkEndpoint { + host: "api.example.com".to_string(), + port: 80, + ports: vec![80, 443], + ..Default::default() + }], + ..Default::default() + }, + ); + + let result = merge_policy( + policy, + &[PolicyMergeOp::RemoveEndpoint { + rule_name: None, + host: "api.example.com".to_string(), + port: 443, + }], + ) + .expect("merge should succeed"); + + let endpoint = &result.policy.network_policies["multi"].endpoints[0]; + assert_eq!(endpoint.ports, vec![80]); + assert_eq!(endpoint.port, 80); + } + + #[test] + fn remove_binary_removes_rule_when_last_binary_is_deleted() { + let mut policy = restrictive_default_policy(); + policy.network_policies.insert( + "github".to_string(), + NetworkPolicyRule { + name: "github".to_string(), + endpoints: vec![endpoint("api.github.com", 443)], + binaries: vec![NetworkBinary { + path: "/usr/bin/gh".to_string(), + ..Default::default() + }], + }, + ); + + let result = merge_policy( + policy, + &[PolicyMergeOp::RemoveBinary { + rule_name: "github".to_string(), + binary_path: "/usr/bin/gh".to_string(), + }], + ) + .expect("merge should succeed"); + + assert!(!result.policy.network_policies.contains_key("github")); + } + + #[test] + fn add_rule_without_existing_match_inserts_requested_key() { + let policy = restrictive_default_policy(); + let result = merge_policy( + policy, + &[PolicyMergeOp::AddRule { + rule_name: "allow_api_example_com_443".to_string(), + rule: rule_with_endpoint("custom", "api.example.com", 443), + }], + ) + .expect("merge should succeed"); + + assert!( + result + .policy + .network_policies + .contains_key("allow_api_example_com_443") + ); + } +} diff --git a/crates/openshell-sandbox/src/grpc_client.rs b/crates/openshell-sandbox/src/grpc_client.rs index 5503637eec..c97d1d7925 100644 --- a/crates/openshell-sandbox/src/grpc_client.rs +++ b/crates/openshell-sandbox/src/grpc_client.rs @@ -133,6 +133,7 @@ async fn sync_policy_with_client( setting_value: None, delete_setting: false, global: false, + merge_operations: vec![], }) .await .into_diagnostic() diff --git a/crates/openshell-server/Cargo.toml b/crates/openshell-server/Cargo.toml index 29f99d0092..b2524ff0b2 100644 --- a/crates/openshell-server/Cargo.toml +++ b/crates/openshell-server/Cargo.toml @@ -17,6 +17,7 @@ path = "src/main.rs" [dependencies] openshell-core = { path = "../openshell-core" } openshell-driver-kubernetes = { path = "../openshell-driver-kubernetes" } +openshell-ocsf = { path = "../openshell-ocsf" } openshell-policy = { path = "../openshell-policy" } openshell-router = { path = "../openshell-router" } diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index 58d0c03cf6..8ef8cb5c73 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -12,8 +12,10 @@ use crate::ServerState; use crate::persistence::{DraftChunkRecord, PolicyRecord, Store}; +use openshell_core::proto::policy_merge_operation; use openshell_core::proto::setting_value; use openshell_core::proto::{ + AddAllowRules as ProtoAddAllowRules, AddDenyRules as ProtoAddDenyRules, ApproveAllDraftChunksRequest, ApproveAllDraftChunksResponse, ApproveDraftChunkRequest, ApproveDraftChunkResponse, ClearDraftChunksRequest, ClearDraftChunksResponse, DraftHistoryEntry, EditDraftChunkRequest, EditDraftChunkResponse, EffectiveSetting, @@ -22,18 +24,29 @@ use openshell_core::proto::{ GetSandboxConfigResponse, GetSandboxLogsRequest, GetSandboxLogsResponse, GetSandboxPolicyStatusRequest, GetSandboxPolicyStatusResponse, GetSandboxProviderEnvironmentRequest, GetSandboxProviderEnvironmentResponse, - ListSandboxPoliciesRequest, ListSandboxPoliciesResponse, PolicyChunk, PolicySource, - PolicyStatus, PushSandboxLogsRequest, PushSandboxLogsResponse, RejectDraftChunkRequest, - RejectDraftChunkResponse, ReportPolicyStatusRequest, ReportPolicyStatusResponse, - SandboxLogLine, SandboxPolicyRevision, SettingScope, SettingValue, SubmitPolicyAnalysisRequest, - SubmitPolicyAnalysisResponse, UndoDraftChunkRequest, UndoDraftChunkResponse, - UpdateConfigRequest, UpdateConfigResponse, + ListSandboxPoliciesRequest, ListSandboxPoliciesResponse, PolicyChunk, PolicyMergeOperation, + PolicySource, PolicyStatus, PushSandboxLogsRequest, PushSandboxLogsResponse, + RejectDraftChunkRequest, RejectDraftChunkResponse, ReportPolicyStatusRequest, + ReportPolicyStatusResponse, SandboxLogLine, SandboxPolicyRevision, SettingScope, SettingValue, + SubmitPolicyAnalysisRequest, SubmitPolicyAnalysisResponse, UndoDraftChunkRequest, + UndoDraftChunkResponse, UpdateConfigRequest, UpdateConfigResponse, }; -use openshell_core::proto::{Sandbox, SandboxPolicy as ProtoSandboxPolicy}; -use openshell_core::settings::{self, SettingValueKind}; +use openshell_core::proto::{ + L7DenyRule, L7Rule, NetworkBinary, NetworkEndpoint, NetworkPolicyRule, Sandbox, + SandboxPolicy as ProtoSandboxPolicy, +}; +use openshell_core::{ + VERSION, + settings::{self, SettingValueKind}, +}; +use openshell_ocsf::{ + ConfigStateChangeBuilder, OCSF_TARGET, OcsfEvent, SandboxContext, SeverityId, StateId, StatusId, +}; +use openshell_policy::{PolicyMergeOp, merge_policy}; use prost::Message; use sha2::{Digest, Sha256}; use std::collections::{BTreeMap, HashMap}; +use std::net::{IpAddr, Ipv4Addr}; use std::sync::Arc; use tonic::{Request, Response, Status}; use tracing::{debug, info, warn}; @@ -61,6 +74,228 @@ const GLOBAL_POLICY_SANDBOX_ID: &str = "__global__"; /// Maximum number of optimistic retry attempts for policy version conflicts. const MERGE_RETRY_LIMIT: usize = 5; +fn emit_gateway_policy_audit_log( + sandbox_id: &str, + sandbox_name: &str, + state_label: &str, + detail: impl Into, + version: i64, + policy_hash: &str, +) { + let message = build_gateway_policy_audit_message( + sandbox_id, + sandbox_name, + state_label, + detail, + version, + policy_hash, + ); + info!( + target: OCSF_TARGET, + sandbox_id = %sandbox_id, + message = %message + ); +} + +fn build_gateway_policy_audit_message( + sandbox_id: &str, + sandbox_name: &str, + state_label: &str, + detail: impl Into, + version: i64, + policy_hash: &str, +) -> String { + let ctx = SandboxContext { + sandbox_id: sandbox_id.to_string(), + sandbox_name: sandbox_name.to_string(), + container_image: "openshell/gateway".to_string(), + hostname: "openshell-gateway".to_string(), + product_version: VERSION.to_string(), + proxy_ip: IpAddr::V4(Ipv4Addr::LOCALHOST), + proxy_port: 0, + }; + let mut builder = ConfigStateChangeBuilder::new(&ctx) + .state(StateId::Other, state_label) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .message(detail.into()); + if version > 0 { + builder = builder.unmapped("policy_version", format!("v{version}")); + } + if !policy_hash.is_empty() { + builder = builder.unmapped("policy_hash", policy_hash.to_string()); + } + let event: OcsfEvent = builder.build(); + event.format_shorthand() +} + +fn summarize_cli_policy_merge_op(operation: &PolicyMergeOp) -> String { + match operation { + PolicyMergeOp::AddRule { rule_name, rule } => summarize_add_endpoint(rule_name, rule), + PolicyMergeOp::RemoveEndpoint { + rule_name, + host, + port, + } => match rule_name { + Some(rule_name) => format!("remove-endpoint {host}:{port} from rule {rule_name}"), + None => format!("remove-endpoint {host}:{port}"), + }, + PolicyMergeOp::RemoveRule { rule_name } => format!("remove-rule {rule_name}"), + PolicyMergeOp::AddDenyRules { + host, + port, + deny_rules, + } => format!( + "add-deny {host}:{port} [{}]", + deny_rules + .iter() + .map(summarize_l7_deny_rule) + .collect::>() + .join(", ") + ), + PolicyMergeOp::AddAllowRules { host, port, rules } => format!( + "add-allow {host}:{port} [{}]", + rules + .iter() + .map(summarize_l7_rule) + .collect::>() + .join(", ") + ), + PolicyMergeOp::RemoveBinary { + rule_name, + binary_path, + } => format!("remove-binary {rule_name} {binary_path}"), + } +} + +fn summarize_add_endpoint(rule_name: &str, rule: &NetworkPolicyRule) -> String { + let endpoints = rule + .endpoints + .iter() + .map(summarize_endpoint) + .collect::>() + .join(", "); + let binaries = summarize_binaries(&rule.binaries); + format!("add-endpoint {rule_name} endpoints=[{endpoints}] binaries=[{binaries}]") +} + +fn summarize_add_rule(rule_name: &str, rule: &NetworkPolicyRule) -> String { + let endpoints = rule + .endpoints + .iter() + .map(summarize_endpoint) + .collect::>() + .join(", "); + let binaries = summarize_binaries(&rule.binaries); + format!("add-rule {rule_name} endpoints=[{endpoints}] binaries=[{binaries}]") +} + +fn summarize_endpoint(endpoint: &NetworkEndpoint) -> String { + let mut parts = vec![format!("{}:{}", endpoint.host, endpoint.port)]; + if !endpoint.protocol.is_empty() { + parts.push(format!("protocol={}", endpoint.protocol)); + } + if !endpoint.access.is_empty() { + parts.push(format!("access={}", endpoint.access)); + } + if !endpoint.enforcement.is_empty() { + parts.push(format!("enforcement={}", endpoint.enforcement)); + } + if !endpoint.tls.is_empty() { + parts.push(format!("tls={}", endpoint.tls)); + } + if !endpoint.allowed_ips.is_empty() { + parts.push(format!("allowed_ips={}", endpoint.allowed_ips.len())); + } + if !endpoint.ports.is_empty() { + parts.push(format!("ports={}", endpoint.ports.len())); + } + if !endpoint.rules.is_empty() { + parts.push(format!( + "allow=[{}]", + endpoint + .rules + .iter() + .map(summarize_l7_rule) + .collect::>() + .join(", ") + )); + } + if !endpoint.deny_rules.is_empty() { + parts.push(format!( + "deny=[{}]", + endpoint + .deny_rules + .iter() + .map(summarize_l7_deny_rule) + .collect::>() + .join(", ") + )); + } + parts.join(" ") +} + +fn summarize_l7_rule(rule: &L7Rule) -> String { + let Some(allow) = rule.allow.as_ref() else { + return "allow".to_string(); + }; + summarize_l7_match( + &allow.method, + &allow.path, + &allow.command, + allow.query.len(), + ) +} + +fn summarize_l7_deny_rule(rule: &L7DenyRule) -> String { + summarize_l7_match(&rule.method, &rule.path, &rule.command, rule.query.len()) +} + +fn summarize_l7_match(method: &str, path: &str, command: &str, query_count: usize) -> String { + let mut parts = Vec::new(); + if !method.is_empty() { + parts.push(method.to_string()); + } + if !path.is_empty() { + parts.push(path.to_string()); + } + if !command.is_empty() { + parts.push(format!("command={}", truncate_for_log(command, 48))); + } + if query_count > 0 { + parts.push(format!("query_keys={query_count}")); + } + if parts.is_empty() { + "rule".to_string() + } else { + parts.join(" ") + } +} + +fn summarize_binaries(binaries: &[NetworkBinary]) -> String { + binaries + .iter() + .map(|binary| binary.path.as_str()) + .collect::>() + .join(", ") +} + +fn summarize_draft_chunk_rule(chunk: &DraftChunkRecord) -> Result { + let rule = NetworkPolicyRule::decode(chunk.proposed_rule.as_slice()) + .map_err(|e| Status::internal(format!("decode proposed_rule failed: {e}")))?; + Ok(summarize_add_rule(&chunk.rule_name, &rule)) +} + +fn truncate_for_log(input: &str, max_chars: usize) -> String { + let mut chars = input.chars(); + let truncated: String = chars.by_ref().take(max_chars).collect(); + if chars.next().is_some() { + format!("{truncated}...") + } else { + truncated + } +} + // --------------------------------------------------------------------------- // Config handlers // --------------------------------------------------------------------------- @@ -242,21 +477,32 @@ pub(super) async fn handle_update_config( let key = req.setting_key.trim(); let has_policy = req.policy.is_some(); let has_setting = !key.is_empty(); + let has_merge_ops = !req.merge_operations.is_empty(); + let mut mutation_count = 0_u8; + mutation_count += u8::from(has_policy); + mutation_count += u8::from(has_setting); + mutation_count += u8::from(has_merge_ops); - if has_policy && has_setting { + if mutation_count > 1 { return Err(Status::invalid_argument( - "policy and setting_key cannot be set in the same request", + "policy, setting_key, and merge_operations are mutually exclusive", )); } - if !has_policy && !has_setting { + if mutation_count == 0 { return Err(Status::invalid_argument( - "either policy or setting_key must be provided", + "one of policy, setting_key, or merge_operations must be provided", )); } if req.global { let _settings_guard = state.settings_mutex.lock().await; + if has_merge_ops { + return Err(Status::invalid_argument( + "merge_operations are not supported for global policy updates", + )); + } + if has_policy { if req.delete_setting { return Err(Status::invalid_argument( @@ -493,6 +739,69 @@ pub(super) async fn handle_update_config( })); } + if has_merge_ops { + let global_settings = load_global_settings(state.store.as_ref()).await?; + if global_settings.settings.contains_key(POLICY_SETTING_KEY) { + return Err(Status::failed_precondition( + "policy is managed globally; delete global policy before sandbox policy update", + )); + } + + let spec = sandbox + .spec + .as_ref() + .ok_or_else(|| Status::internal("sandbox has no spec"))?; + let merge_ops = parse_merge_operations(&req.merge_operations)?; + validate_merge_operations_for_server(&merge_ops)?; + let (version, hash) = apply_merge_operations_with_retry( + state.store.as_ref(), + &sandbox_id, + spec.policy.as_ref(), + &merge_ops, + ) + .await?; + + state.sandbox_watch_bus.notify(&sandbox_id); + emit_gateway_policy_audit_log( + &sandbox_id, + &sandbox.name, + "merged", + format!( + "gateway merged {} incremental policy operation(s)", + merge_ops.len() + ), + version, + &hash, + ); + for operation in &merge_ops { + emit_gateway_policy_audit_log( + &sandbox_id, + &sandbox.name, + "merged", + format!( + "gateway merged incremental policy op: {}", + summarize_cli_policy_merge_op(operation) + ), + version, + &hash, + ); + } + info!( + sandbox_id = %sandbox_id, + version, + policy_hash = %hash, + operation_count = merge_ops.len(), + "UpdateConfig: merged incremental policy operations" + ); + + return Ok(Response::new(UpdateConfigResponse { + version: u32::try_from(version).unwrap_or(0), + policy_hash: hash, + settings_revision: 0, + deleted: false, + })); + } + // Sandbox-scoped policy update. let mut new_policy = req .policy @@ -1045,6 +1354,7 @@ pub(super) async fn handle_approve_draft_chunk( let (version, hash) = merge_chunk_into_policy(state.store.as_ref(), &sandbox_id, &chunk).await?; + let chunk_summary = summarize_draft_chunk_rule(&chunk)?; let now_ms = current_time_ms().map_err(|e| Status::internal(format!("timestamp error: {e}")))?; @@ -1055,6 +1365,17 @@ pub(super) async fn handle_approve_draft_chunk( .map_err(|e| Status::internal(format!("update chunk status failed: {e}")))?; state.sandbox_watch_bus.notify(&sandbox_id); + emit_gateway_policy_audit_log( + &sandbox_id, + &sandbox.name, + "approved", + format!( + "gateway approved draft chunk {}: {chunk_summary}", + req.chunk_id + ), + version, + &hash, + ); info!( sandbox_id = %sandbox_id, @@ -1120,7 +1441,18 @@ pub(super) async fn handle_reject_draft_chunk( if was_approved { require_no_global_policy(state).await?; - remove_chunk_from_policy(state, &sandbox_id, &chunk).await?; + let (version, hash) = remove_chunk_from_policy(state, &sandbox_id, &chunk).await?; + emit_gateway_policy_audit_log( + &sandbox_id, + &sandbox.name, + "removed", + format!( + "gateway removed previously approved draft chunk {}: remove-binary {} {}", + req.chunk_id, chunk.rule_name, chunk.binary + ), + version, + &hash, + ); } let now_ms = @@ -1203,6 +1535,7 @@ pub(super) async fn handle_approve_all_draft_chunks( merge_chunk_into_policy(state.store.as_ref(), &sandbox_id, chunk).await?; last_version = version; last_hash = hash; + let chunk_summary = summarize_draft_chunk_rule(chunk)?; let now_ms = current_time_ms().map_err(|e| Status::internal(format!("timestamp error: {e}")))?; @@ -1212,10 +1545,28 @@ pub(super) async fn handle_approve_all_draft_chunks( .await .map_err(|e| Status::internal(format!("update chunk status failed: {e}")))?; + emit_gateway_policy_audit_log( + &sandbox_id, + &sandbox.name, + "approved", + format!("gateway approved draft chunk {}: {chunk_summary}", chunk.id), + version, + &last_hash, + ); chunks_approved += 1; } state.sandbox_watch_bus.notify(&sandbox_id); + emit_gateway_policy_audit_log( + &sandbox_id, + &sandbox.name, + "merged", + format!( + "gateway bulk-approved {chunks_approved} draft chunk(s) and skipped {chunks_skipped}" + ), + last_version, + &last_hash, + ); info!( sandbox_id = %sandbox_id, @@ -1337,6 +1688,17 @@ pub(super) async fn handle_undo_draft_chunk( .map_err(|e| Status::internal(format!("update chunk status failed: {e}")))?; state.sandbox_watch_bus.notify(&sandbox_id); + emit_gateway_policy_audit_log( + &sandbox_id, + &sandbox.name, + "removed", + format!( + "gateway reverted approved draft chunk {}: remove-binary {} {}", + req.chunk_id, chunk.rule_name, chunk.binary + ), + version, + &hash, + ); info!( sandbox_id = %sandbox_id, @@ -1615,9 +1977,7 @@ fn generate_security_notes(host: &str, port: u16) -> String { /// /// This is defense-in-depth: the proxy blocks these at runtime, so /// merging them into the active policy would be silently un-enforceable. -fn validate_rule_not_always_blocked( - rule: &openshell_core::proto::NetworkPolicyRule, -) -> Result<(), Status> { +fn validate_rule_not_always_blocked(rule: &NetworkPolicyRule) -> Result<(), Status> { use openshell_core::net::{is_always_blocked_ip, is_always_blocked_net}; use std::net::IpAddr; @@ -1677,90 +2037,203 @@ async fn require_no_global_policy(state: &ServerState) -> Result<(), Status> { Ok(()) } -pub(super) async fn merge_chunk_into_policy( - store: &Store, - sandbox_id: &str, - chunk: &DraftChunkRecord, -) -> Result<(i64, String), Status> { - use openshell_core::proto::NetworkPolicyRule; +fn parse_merge_operations( + proto_ops: &[PolicyMergeOperation], +) -> Result, Status> { + proto_ops + .iter() + .enumerate() + .map(|(index, operation)| { + let Some(operation) = operation.operation.as_ref() else { + return Err(Status::invalid_argument(format!( + "merge_operations[{index}] is missing an operation" + ))); + }; - let rule = NetworkPolicyRule::decode(chunk.proposed_rule.as_slice()) - .map_err(|e| Status::internal(format!("decode proposed_rule failed: {e}")))?; + match operation { + policy_merge_operation::Operation::AddRule(add_rule) => { + let rule_name = add_rule.rule_name.trim(); + if rule_name.is_empty() { + return Err(Status::invalid_argument(format!( + "merge_operations[{index}].add_rule.rule_name is required" + ))); + } + if add_rule.rule.as_ref().is_none_or(|rule| rule.endpoints.is_empty()) { + return Err(Status::invalid_argument(format!( + "merge_operations[{index}].add_rule.rule must contain at least one endpoint" + ))); + } + Ok(PolicyMergeOp::AddRule { + rule_name: rule_name.to_string(), + rule: add_rule.rule.clone().unwrap_or_default(), + }) + } + policy_merge_operation::Operation::RemoveEndpoint(remove_endpoint) => { + if remove_endpoint.host.trim().is_empty() || remove_endpoint.port == 0 { + return Err(Status::invalid_argument(format!( + "merge_operations[{index}].remove_endpoint requires host and non-zero port" + ))); + } + let rule_name = if remove_endpoint.rule_name.trim().is_empty() { + None + } else { + Some(remove_endpoint.rule_name.trim().to_string()) + }; + Ok(PolicyMergeOp::RemoveEndpoint { + rule_name, + host: remove_endpoint.host.trim().to_string(), + port: remove_endpoint.port, + }) + } + policy_merge_operation::Operation::RemoveRule(remove_rule) => { + let rule_name = remove_rule.rule_name.trim(); + if rule_name.is_empty() { + return Err(Status::invalid_argument(format!( + "merge_operations[{index}].remove_rule.rule_name is required" + ))); + } + Ok(PolicyMergeOp::RemoveRule { + rule_name: rule_name.to_string(), + }) + } + policy_merge_operation::Operation::AddDenyRules(add_deny_rules) => { + parse_proto_add_deny_rules(index, add_deny_rules) + } + policy_merge_operation::Operation::AddAllowRules(add_allow_rules) => { + parse_proto_add_allow_rules(index, add_allow_rules) + } + policy_merge_operation::Operation::RemoveBinary(remove_binary) => { + let rule_name = remove_binary.rule_name.trim(); + let binary_path = remove_binary.binary_path.trim(); + if rule_name.is_empty() || binary_path.is_empty() { + return Err(Status::invalid_argument(format!( + "merge_operations[{index}].remove_binary requires rule_name and binary_path" + ))); + } + Ok(PolicyMergeOp::RemoveBinary { + rule_name: rule_name.to_string(), + binary_path: binary_path.to_string(), + }) + } + } + }) + .collect() +} + +fn parse_proto_add_deny_rules( + index: usize, + add_deny_rules: &ProtoAddDenyRules, +) -> Result { + if add_deny_rules.host.trim().is_empty() + || add_deny_rules.port == 0 + || add_deny_rules.deny_rules.is_empty() + { + return Err(Status::invalid_argument(format!( + "merge_operations[{index}].add_deny_rules requires host, non-zero port, and at least one deny rule" + ))); + } + + Ok(PolicyMergeOp::AddDenyRules { + host: add_deny_rules.host.trim().to_string(), + port: add_deny_rules.port, + deny_rules: add_deny_rules.deny_rules.clone(), + }) +} + +fn parse_proto_add_allow_rules( + index: usize, + add_allow_rules: &ProtoAddAllowRules, +) -> Result { + if add_allow_rules.host.trim().is_empty() + || add_allow_rules.port == 0 + || add_allow_rules.rules.is_empty() + { + return Err(Status::invalid_argument(format!( + "merge_operations[{index}].add_allow_rules requires host, non-zero port, and at least one allow rule" + ))); + } + if add_allow_rules + .rules + .iter() + .any(|rule| rule.allow.as_ref().is_none()) + { + return Err(Status::invalid_argument(format!( + "merge_operations[{index}].add_allow_rules rules must include allow payloads" + ))); + } + + Ok(PolicyMergeOp::AddAllowRules { + host: add_allow_rules.host.trim().to_string(), + port: add_allow_rules.port, + rules: add_allow_rules.rules.clone(), + }) +} - // Defense-in-depth: reject proposed rules targeting always-blocked - // destinations. Even if the sandbox mapper didn't filter these (e.g., - // an older sandbox version), the proxy will deny them at runtime. - validate_rule_not_always_blocked(&rule)?; +fn validate_merge_operations_for_server(operations: &[PolicyMergeOp]) -> Result<(), Status> { + for operation in operations { + if let PolicyMergeOp::AddRule { rule, .. } = operation { + validate_rule_not_always_blocked(rule)?; + } + } + Ok(()) +} +fn map_policy_merge_error(error: openshell_policy::PolicyMergeError) -> Status { + match error { + openshell_policy::PolicyMergeError::MissingRuleNameForAddRule + | openshell_policy::PolicyMergeError::InvalidEndpointReference { .. } + | openshell_policy::PolicyMergeError::UnsupportedAccessPreset { .. } => { + Status::invalid_argument(error.to_string()) + } + openshell_policy::PolicyMergeError::EndpointNotFound { .. } + | openshell_policy::PolicyMergeError::EndpointHasNoL7Inspection { .. } + | openshell_policy::PolicyMergeError::UnsupportedEndpointProtocol { .. } + | openshell_policy::PolicyMergeError::EndpointHasNoAllowBase { .. } => { + Status::failed_precondition(error.to_string()) + } + } +} + +async fn apply_merge_operations_with_retry( + store: &Store, + sandbox_id: &str, + baseline_policy: Option<&ProtoSandboxPolicy>, + operations: &[PolicyMergeOp], +) -> Result<(i64, String), Status> { for attempt in 1..=MERGE_RETRY_LIMIT { let latest = store .get_latest_policy(sandbox_id) .await .map_err(|e| Status::internal(format!("fetch latest policy failed: {e}")))?; - let mut policy = if let Some(ref record) = latest { + let current_policy = if let Some(ref record) = latest { ProtoSandboxPolicy::decode(record.policy_payload.as_slice()) .map_err(|e| Status::internal(format!("decode current policy failed: {e}")))? } else { - ProtoSandboxPolicy::default() + baseline_policy.cloned().unwrap_or_default() }; - let base_version = latest.as_ref().map_or(0, |r| r.version); + let merged = merge_policy(current_policy, operations).map_err(map_policy_merge_error)?; + let new_policy = merged.policy; + let hash = deterministic_policy_hash(&new_policy); - let chunk_host_lc = chunk.host.to_lowercase(); - let chunk_port = chunk.port as u32; + if let Some(baseline_policy) = baseline_policy { + validate_static_fields_unchanged(baseline_policy, &new_policy)?; + } + validate_policy_safety(&new_policy)?; - let merge_key = if policy.network_policies.contains_key(&chunk.rule_name) { - Some(chunk.rule_name.clone()) - } else { - policy - .network_policies - .iter() - .find_map(|(key, existing_rule)| { - let has_match = existing_rule.endpoints.iter().any(|ep| { - let host_match = ep.host.to_lowercase() == chunk_host_lc; - let port_match = if ep.ports.is_empty() { - ep.port == chunk_port - } else { - ep.ports.contains(&chunk_port) - }; - host_match && port_match - }); - has_match.then(|| key.clone()) - }) - }; + if let Some(ref current) = latest + && current.policy_hash == hash + { + return Ok((current.version, hash)); + } - if let Some(key) = merge_key { - let existing = policy.network_policies.get_mut(&key).unwrap(); - for b in &rule.binaries { - if !existing.binaries.iter().any(|eb| eb.path == b.path) { - existing.binaries.push(b.clone()); - } - } - for ep in &rule.endpoints { - if let Some(existing_ep) = existing.endpoints.iter_mut().find(|e| { - e.host.to_lowercase() == ep.host.to_lowercase() - && (e.port == ep.port - || (!e.ports.is_empty() && e.ports.contains(&ep.port))) - }) { - for ip in &ep.allowed_ips { - if !existing_ep.allowed_ips.contains(ip) { - existing_ep.allowed_ips.push(ip.clone()); - } - } - } else { - existing.endpoints.push(ep.clone()); - } - } - } else { - policy - .network_policies - .insert(chunk.rule_name.clone(), rule.clone()); + if latest.is_none() && !merged.changed { + return Ok((0, hash)); } - let payload = policy.encode_to_vec(); - let hash = deterministic_policy_hash(&policy); - let next_version = base_version + 1; + let payload = new_policy.encode_to_vec(); + let next_version = latest.as_ref().map_or(1, |record| record.version + 1); let policy_id = uuid::Uuid::new_v4().to_string(); match store @@ -1775,10 +2248,10 @@ pub(super) async fn merge_chunk_into_policy( if attempt > 1 { info!( sandbox_id = %sandbox_id, - rule_name = %chunk.rule_name, attempt, version = next_version, - "merge_chunk_into_policy: succeeded after version conflict retry" + operation_count = operations.len(), + "apply_merge_operations_with_retry: succeeded after version conflict retry" ); } @@ -1789,10 +2262,10 @@ pub(super) async fn merge_chunk_into_policy( if msg.contains("UNIQUE") || msg.contains("unique") || msg.contains("duplicate") { warn!( sandbox_id = %sandbox_id, - rule_name = %chunk.rule_name, attempt, conflicting_version = next_version, - "merge_chunk_into_policy: version conflict, retrying" + operation_count = operations.len(), + "apply_merge_operations_with_retry: version conflict, retrying" ); tokio::task::yield_now().await; continue; @@ -1805,90 +2278,44 @@ pub(super) async fn merge_chunk_into_policy( } Err(Status::aborted(format!( - "merge_chunk_into_policy: gave up after {} version conflict retries for rule '{}'", - MERGE_RETRY_LIMIT, chunk.rule_name + "apply_merge_operations_with_retry: gave up after {MERGE_RETRY_LIMIT} version conflict retries" ))) } +pub(super) async fn merge_chunk_into_policy( + store: &Store, + sandbox_id: &str, + chunk: &DraftChunkRecord, +) -> Result<(i64, String), Status> { + let rule = NetworkPolicyRule::decode(chunk.proposed_rule.as_slice()) + .map_err(|e| Status::internal(format!("decode proposed_rule failed: {e}")))?; + apply_merge_operations_with_retry( + store, + sandbox_id, + None, + &[PolicyMergeOp::AddRule { + rule_name: chunk.rule_name.clone(), + rule, + }], + ) + .await +} + async fn remove_chunk_from_policy( state: &ServerState, sandbox_id: &str, chunk: &DraftChunkRecord, ) -> Result<(i64, String), Status> { - for attempt in 1..=MERGE_RETRY_LIMIT { - let latest = state - .store - .get_latest_policy(sandbox_id) - .await - .map_err(|e| Status::internal(format!("fetch latest policy failed: {e}")))? - .ok_or_else(|| Status::internal("no active policy to undo from"))?; - - let mut policy = ProtoSandboxPolicy::decode(latest.policy_payload.as_slice()) - .map_err(|e| Status::internal(format!("decode current policy failed: {e}")))?; - - let should_remove = - if let Some(existing) = policy.network_policies.get_mut(&chunk.rule_name) { - existing.binaries.retain(|b| b.path != chunk.binary); - existing.binaries.is_empty() - } else { - false - }; - if should_remove { - policy.network_policies.remove(&chunk.rule_name); - } - - let payload = policy.encode_to_vec(); - let hash = deterministic_policy_hash(&policy); - let next_version = latest.version + 1; - let policy_id = uuid::Uuid::new_v4().to_string(); - - match state - .store - .put_policy_revision(&policy_id, sandbox_id, next_version, &payload, &hash) - .await - { - Ok(()) => { - let _ = state - .store - .supersede_older_policies(sandbox_id, next_version) - .await; - - if attempt > 1 { - info!( - sandbox_id = %sandbox_id, - rule_name = %chunk.rule_name, - attempt, - version = next_version, - "remove_chunk_from_policy: succeeded after version conflict retry" - ); - } - - return Ok((next_version, hash)); - } - Err(e) => { - let msg = e.to_string(); - if msg.contains("UNIQUE") || msg.contains("unique") || msg.contains("duplicate") { - warn!( - sandbox_id = %sandbox_id, - rule_name = %chunk.rule_name, - attempt, - conflicting_version = next_version, - "remove_chunk_from_policy: version conflict, retrying" - ); - tokio::task::yield_now().await; - continue; - } - return Err(Status::internal(format!( - "persist policy revision failed: {e}" - ))); - } - } - } - - Err(Status::aborted(format!( - "remove_chunk_from_policy: gave up after {} version conflict retries for rule '{}'", - MERGE_RETRY_LIMIT, chunk.rule_name - ))) + apply_merge_operations_with_retry( + state.store.as_ref(), + sandbox_id, + None, + &[PolicyMergeOp::RemoveBinary { + rule_name: chunk.rule_name.clone(), + binary_path: chunk.binary.clone(), + }], + ) + .await } // --------------------------------------------------------------------------- @@ -2151,6 +2578,7 @@ mod tests { use super::*; use crate::persistence::Store; use std::collections::HashMap; + use std::sync::Arc; use tonic::Code; // ---- Sandbox without policy ---- @@ -2184,9 +2612,7 @@ mod tests { #[tokio::test] async fn sandbox_policy_backfill_on_update_when_no_baseline() { - use openshell_core::proto::{ - FilesystemPolicy, LandlockPolicy, ProcessPolicy, SandboxPhase, SandboxSpec, - }; + use openshell_core::proto::{FilesystemPolicy, LandlockPolicy, SandboxPhase, SandboxSpec}; let store = Store::connect("sqlite::memory:").await.unwrap(); @@ -2241,6 +2667,71 @@ mod tests { assert_eq!(policy.process.unwrap().run_as_user, "sandbox"); } + #[test] + fn build_gateway_policy_audit_message_formats_ocsf_config_line() { + let message = build_gateway_policy_audit_message( + "sb-123", + "demo-sandbox", + "merged", + "gateway merged incremental policy op: add-allow api.github.com:443 [POST /repos/*/issues]", + 7, + "sha256:testhash", + ); + + assert_eq!( + message, + "CONFIG:MERGED [INFO] gateway merged incremental policy op: add-allow api.github.com:443 [POST /repos/*/issues] [version:v7 hash:sha256:testhash]" + ); + } + + #[test] + fn summarize_cli_policy_merge_op_formats_rest_allow_rules() { + let operation = PolicyMergeOp::AddAllowRules { + host: "api.github.com".to_string(), + port: 443, + rules: vec![L7Rule { + allow: Some(openshell_core::proto::L7Allow { + method: "POST".to_string(), + path: "/repos/*/issues".to_string(), + command: String::new(), + query: HashMap::new(), + }), + }], + }; + + assert_eq!( + summarize_cli_policy_merge_op(&operation), + "add-allow api.github.com:443 [POST /repos/*/issues]" + ); + } + + #[test] + fn summarize_cli_policy_merge_op_formats_endpoint_additions() { + let operation = PolicyMergeOp::AddRule { + rule_name: "github_api".to_string(), + rule: NetworkPolicyRule { + name: "github_api".to_string(), + endpoints: vec![NetworkEndpoint { + host: "api.github.com".to_string(), + port: 443, + protocol: "rest".to_string(), + access: "read-only".to_string(), + enforcement: "enforce".to_string(), + ..Default::default() + }], + binaries: vec![NetworkBinary { + path: "/usr/bin/curl".to_string(), + ..Default::default() + }], + }, + }; + + assert_eq!( + summarize_cli_policy_merge_op(&operation), + "add-endpoint github_api endpoints=[api.github.com:443 protocol=rest access=read-only enforcement=enforce] binaries=[/usr/bin/curl]" + ); + } + // ---- merge_chunk_into_policy ---- #[tokio::test] @@ -2488,6 +2979,89 @@ mod tests { assert!(policy.network_policies.contains_key("allow_10_0_0_5_8080")); } + #[tokio::test] + async fn concurrent_merge_batches_preserve_both_updates() { + use openshell_core::proto::{ + L7Allow, L7DenyRule, L7Rule, NetworkEndpoint, NetworkPolicyRule, SandboxPolicy, + }; + + let store = Store::connect("sqlite::memory:").await.unwrap(); + let sandbox_id = "sb-concurrent-merge"; + + let initial_policy = SandboxPolicy { + network_policies: [( + "github".to_string(), + NetworkPolicyRule { + name: "github".to_string(), + endpoints: vec![NetworkEndpoint { + host: "api.github.com".to_string(), + port: 443, + ports: vec![443], + protocol: "rest".to_string(), + access: "read-only".to_string(), + ..Default::default() + }], + ..Default::default() + }, + )] + .into_iter() + .collect(), + ..Default::default() + }; + store + .put_policy_revision( + "p-seed", + sandbox_id, + 1, + &initial_policy.encode_to_vec(), + "seed-hash", + ) + .await + .unwrap(); + + let add_allow = [PolicyMergeOp::AddAllowRules { + host: "api.github.com".to_string(), + port: 443, + rules: vec![L7Rule { + allow: Some(L7Allow { + method: "POST".to_string(), + path: "/repos/*/issues".to_string(), + command: String::new(), + query: HashMap::new(), + }), + }], + }]; + let add_deny = [PolicyMergeOp::AddDenyRules { + host: "api.github.com".to_string(), + port: 443, + deny_rules: vec![L7DenyRule { + method: "POST".to_string(), + path: "/admin".to_string(), + query: HashMap::new(), + ..Default::default() + }], + }]; + + let (left, right) = tokio::join!( + apply_merge_operations_with_retry(&store, sandbox_id, None, &add_allow), + apply_merge_operations_with_retry(&store, sandbox_id, None, &add_deny), + ); + + let mut versions = vec![left.unwrap().0, right.unwrap().0]; + versions.sort_unstable(); + assert_eq!(versions, vec![2, 3]); + + let latest = store.get_latest_policy(sandbox_id).await.unwrap().unwrap(); + assert_eq!(latest.version, 3); + + let policy = SandboxPolicy::decode(latest.policy_payload.as_slice()).unwrap(); + let endpoint = &policy.network_policies["github"].endpoints[0]; + assert!(endpoint.access.is_empty()); + assert_eq!(endpoint.rules.len(), 4); + assert_eq!(endpoint.deny_rules.len(), 1); + assert_eq!(endpoint.deny_rules[0].path, "/admin"); + } + // ---- validate_rule_not_always_blocked ---- #[test] @@ -2608,7 +3182,7 @@ mod tests { let global = StoredSettings::default(); let sandbox = StoredSettings::default(); let merged = merge_effective_settings(&global, &sandbox).unwrap(); - for registered in openshell_core::settings::REGISTERED_SETTINGS { + for registered in settings::REGISTERED_SETTINGS { let setting = merged .get(registered.key) .unwrap_or_else(|| panic!("missing registered key {}", registered.key)); @@ -2625,7 +3199,7 @@ mod tests { fn materialize_global_settings_includes_unset_registered_keys() { let global = StoredSettings::default(); let materialized = materialize_global_settings(&global).unwrap(); - for registered in openshell_core::settings::REGISTERED_SETTINGS { + for registered in settings::REGISTERED_SETTINGS { let setting = materialized .get(registered.key) .unwrap_or_else(|| panic!("missing registered key {}", registered.key)); @@ -2785,7 +3359,7 @@ mod tests { let global = StoredSettings::default(); let sandbox = StoredSettings::default(); let merged = merge_effective_settings(&global, &sandbox).unwrap(); - for registered in openshell_core::settings::REGISTERED_SETTINGS { + for registered in settings::REGISTERED_SETTINGS { let setting = merged.get(registered.key).unwrap(); assert_eq!(setting.scope, SettingScope::Unspecified as i32); assert!(setting.value.is_none()); @@ -3066,12 +3640,12 @@ mod tests { #[tokio::test] async fn concurrent_global_setting_mutations_are_serialized() { - let store = std::sync::Arc::new( + let store = Arc::new( Store::connect("sqlite::memory:?cache=shared") .await .unwrap(), ); - let mutex = std::sync::Arc::new(tokio::sync::Mutex::new(())); + let mutex = Arc::new(tokio::sync::Mutex::new(())); let n = 50; let mut handles = Vec::with_capacity(n); @@ -3101,7 +3675,7 @@ mod tests { #[tokio::test] async fn concurrent_global_setting_mutations_without_lock_can_lose_writes() { - let store = std::sync::Arc::new( + let store = Arc::new( Store::connect("sqlite::memory:?cache=shared") .await .unwrap(), diff --git a/crates/openshell-server/src/grpc/validation.rs b/crates/openshell-server/src/grpc/validation.rs index 8d53da6a0a..1517c0577b 100644 --- a/crates/openshell-server/src/grpc/validation.rs +++ b/crates/openshell-server/src/grpc/validation.rs @@ -382,6 +382,7 @@ pub(super) fn level_matches(log_level: &str, min_level: &str) -> bool { "ERROR" => 0, "WARN" => 1, "INFO" => 2, + "OCSF" => 2, "DEBUG" => 3, "TRACE" => 4, _ => 5, // unknown levels always pass @@ -413,6 +414,12 @@ mod tests { SandboxSpec::default() } + #[test] + fn level_matches_treats_ocsf_as_info() { + assert!(level_matches("OCSF", "INFO")); + assert!(!level_matches("OCSF", "WARN")); + } + #[test] fn validate_sandbox_spec_accepts_gpu_flag() { let spec = SandboxSpec { diff --git a/crates/openshell-server/src/tracing_bus.rs b/crates/openshell-server/src/tracing_bus.rs index 6767a450ee..cf168e3063 100644 --- a/crates/openshell-server/src/tracing_bus.rs +++ b/crates/openshell-server/src/tracing_bus.rs @@ -8,6 +8,7 @@ use std::sync::{Arc, Mutex}; use std::time::{SystemTime, UNIX_EPOCH}; use openshell_core::proto::{SandboxLogLine, SandboxStreamEvent}; +use openshell_ocsf::OCSF_TARGET; use tokio::sync::broadcast; use tracing::{Event, Subscriber}; use tracing_subscriber::layer::Context; @@ -147,12 +148,13 @@ where }; let msg = visitor.message.unwrap_or_else(|| meta.name().to_string()); + let level = display_level(meta.target(), &meta.level().to_string()); let ts = current_time_ms().unwrap_or(0); let log = SandboxLogLine { sandbox_id: sandbox_id.clone(), timestamp_ms: ts, - level: meta.level().to_string(), + level, target: meta.target().to_string(), message: msg, source: "gateway".to_string(), @@ -196,6 +198,14 @@ fn current_time_ms() -> Option { i64::try_from(now.as_millis()).ok() } +fn display_level(target: &str, level: &str) -> String { + if target == OCSF_TARGET { + "OCSF".to_string() + } else { + level.to_string() + } +} + #[cfg(test)] mod tests { use super::*; @@ -274,6 +284,12 @@ mod tests { bus.remove("nonexistent"); } + #[test] + fn display_level_maps_ocsf_target_to_ocsf() { + assert_eq!(display_level(OCSF_TARGET, "INFO"), "OCSF"); + assert_eq!(display_level("openshell_server", "WARN"), "WARN"); + } + #[test] fn platform_event_bus_remove_cleans_up() { let bus = PlatformEventBus::new(); diff --git a/crates/openshell-tui/src/lib.rs b/crates/openshell-tui/src/lib.rs index f187f59fb4..63cfb79d6d 100644 --- a/crates/openshell-tui/src/lib.rs +++ b/crates/openshell-tui/src/lib.rs @@ -1960,6 +1960,7 @@ fn spawn_set_global_setting(app: &App, tx: mpsc::UnboundedSender) { setting_value: Some(SettingValue { value: Some(value) }), delete_setting: false, global: true, + merge_operations: vec![], }; let result = tokio::time::timeout(Duration::from_secs(5), client.update_config(req)).await; @@ -1994,6 +1995,7 @@ fn spawn_delete_global_setting(app: &App, tx: mpsc::UnboundedSender) { setting_value: None, delete_setting: true, global: true, + merge_operations: vec![], }; let result = tokio::time::timeout(Duration::from_secs(5), client.update_config(req)).await; @@ -2062,6 +2064,7 @@ fn spawn_set_sandbox_setting(app: &App, tx: mpsc::UnboundedSender) { setting_value: Some(SettingValue { value: Some(value) }), delete_setting: false, global: false, + merge_operations: vec![], }; let result = tokio::time::timeout(Duration::from_secs(5), client.update_config(req)).await; @@ -2100,6 +2103,7 @@ fn spawn_delete_sandbox_setting(app: &App, tx: mpsc::UnboundedSender) { setting_value: None, delete_setting: true, global: false, + merge_operations: vec![], }; let result = tokio::time::timeout(Duration::from_secs(5), client.update_config(req)).await; diff --git a/deploy/docker/Dockerfile.images b/deploy/docker/Dockerfile.images index d29fa3d7f2..a85a6a9ae3 100644 --- a/deploy/docker/Dockerfile.images +++ b/deploy/docker/Dockerfile.images @@ -115,6 +115,7 @@ ARG OPENSHELL_CARGO_VERSION COPY crates/openshell-core/ crates/openshell-core/ COPY crates/openshell-driver-kubernetes/ crates/openshell-driver-kubernetes/ +COPY crates/openshell-ocsf/ crates/openshell-ocsf/ COPY crates/openshell-policy/ crates/openshell-policy/ COPY crates/openshell-providers/ crates/openshell-providers/ COPY crates/openshell-router/ crates/openshell-router/ diff --git a/docs/observability/accessing-logs.mdx b/docs/observability/accessing-logs.mdx index e3599a68f8..1995ab9d7b 100644 --- a/docs/observability/accessing-logs.mdx +++ b/docs/observability/accessing-logs.mdx @@ -27,6 +27,8 @@ The CLI receives logs from the gateway over gRPC. Each line includes a timestamp OCSF structured events show `OCSF` as the level. Standard tracing events show `INFO`, `WARN`, or `ERROR`. +Gateway-originated policy mutations also appear in this stream. When the gateway merges `openshell policy update` operations or approves or removes draft policy chunks, it emits `gateway` `OCSF` `CONFIG:*` lines for the affected sandbox so you can see the exact logical change that produced a new policy revision. + ## TUI The TUI dashboard displays sandbox logs in real time. Logs appear in the log panel with the same format as the CLI. diff --git a/docs/reference/policy-schema.mdx b/docs/reference/policy-schema.mdx index 3e505cf3e0..7152731cf4 100644 --- a/docs/reference/policy-schema.mdx +++ b/docs/reference/policy-schema.mdx @@ -30,7 +30,7 @@ network_policies: { ... } | `process` | object | No | Static | Sets the user and group the agent process runs as. | | `network_policies` | map | No | Dynamic | Declares which binaries can reach which network endpoints. | -Static fields are set at sandbox creation time. Changing them requires destroying and recreating the sandbox. Dynamic fields can be updated on a running sandbox with `openshell policy set` and take effect without restarting. +Static fields are set at sandbox creation time. Changing them requires destroying and recreating the sandbox. Dynamic fields can be updated on a running sandbox with `openshell policy update` for incremental merges or `openshell policy set` for full replacement, and take effect without restarting. ## Version diff --git a/docs/sandboxes/policies.mdx b/docs/sandboxes/policies.mdx index 8d4831f1b0..9e8d5fafe3 100644 --- a/docs/sandboxes/policies.mdx +++ b/docs/sandboxes/policies.mdx @@ -47,7 +47,7 @@ network_policies: ``` Static sections are locked at sandbox creation. Changing them requires destroying and recreating the sandbox. -Dynamic sections can be updated on a running sandbox with `openshell policy set` and take effect without restarting. +Dynamic sections can be updated on a running sandbox with `openshell policy update` for incremental merges or `openshell policy set` for full replacement, and take effect without restarting. | Section | Type | Description | |---|---|---| @@ -121,26 +121,265 @@ The following steps outline the hot-reload policy update workflow. openshell logs --tail --source sandbox ``` -3. Pull the current policy. Strip the metadata header (Version, Hash, Status) before reusing the file. +3. For additive network changes, use `openshell policy update`. This is the fastest path for adding endpoints, binaries, or REST allow/deny rules without replacing the full policy. The full option and format reference is in [Incremental Policy Updates](#incremental-policy-updates). + + ```shell + openshell policy update \ + --add-endpoint api.github.com:443:read-only:rest:enforce \ + --binary /usr/bin/gh \ + --wait + + openshell policy update \ + --add-allow 'api.github.com:443:POST:/repos/*/issues' \ + --wait + ``` + + `--add-allow` and `--add-deny` currently target existing `protocol: rest` endpoints only. If you pass multiple update flags in one command, OpenShell applies them as one atomic merge batch and persists at most one new revision. + +4. For larger edits, pull the current policy and edit the YAML directly. Strip the metadata header (Version, Hash, Status) before reusing the file. ```shell openshell policy get --full > current-policy.yaml ``` -4. Edit the YAML: add or adjust `network_policies` entries, binaries, `access`, or `rules`. +5. Edit the YAML: add or adjust `network_policies` entries, binaries, `access`, or `rules`. -5. Push the updated policy. Exit codes: 0 = loaded, 1 = validation failed, 124 = timeout. +6. Push the updated policy when you need a full replacement. Exit codes: 0 = loaded, 1 = validation failed, 124 = timeout. ```shell openshell policy set --policy current-policy.yaml --wait ``` -6. Verify the new revision. If status is `loaded`, repeat from step 2 as needed; if `failed`, fix the policy and repeat from step 4. +7. Verify the new revision. If status is `loaded`, repeat from step 2 as needed; if `failed`, fix the policy and repeat from step 4. ```shell openshell policy list ``` +## Incremental Policy Updates + +Use `openshell policy update` when you want to merge network policy changes into the current live policy instead of replacing the whole YAML document. This command only updates the dynamic `network_policies` section. + +`openshell policy update` is useful when you want to: + +- add a new endpoint for an existing binary without touching other policy sections. +- add a few REST allow or deny rules after you see a blocked request in the logs. +- remove one endpoint or one named rule without rewriting the rest of the file. +- preview a merged result locally with `--dry-run` before you send it to the gateway. + +Use `openshell policy set` instead when you want to replace the full policy, update static sections, or make broader edits that are easier to express in YAML. + +### Update Commands + +The incremental update surface is split into endpoint-level operations and REST rule-level operations. + +| Flag | What it changes | Typical use | +|---|---|---| +| `--add-endpoint ` | Creates or merges a network rule and endpoint. | Allow a new host and port, optionally with `access`, `protocol`, `enforcement`, and binaries. | +| `--remove-endpoint ` | Removes one host and port match from the current policy. | Drop a stale endpoint or remove one port from a multi-port endpoint. | +| `--remove-rule ` | Deletes a named `network_policies` entry. | Remove a whole rule by name when you no longer need it. | +| `--add-allow ` | Appends REST allow rules to an existing endpoint. | Permit one additional method and path on a REST API that is already configured. | +| `--add-deny ` | Appends REST deny rules to an existing endpoint. | Block a sensitive REST path under an endpoint that is otherwise allowed. | +| `--binary ` | Adds binaries to every `--add-endpoint` rule in the same command. | Bind a new endpoint to one or more executables. | +| `--rule-name ` | Overrides the generated rule name. | Keep a stable human-chosen rule name when adding exactly one endpoint. | +| `--dry-run` | Shows the merged policy locally and does not call the gateway. | Review the result before persisting it. | +| `--wait` | Polls until the sandbox reports that the new revision loaded. | Confirm the change took effect before continuing. | +| `--timeout ` | Sets the timeout for `--wait`. | Extend the wait window for slower sandboxes. | + +`--wait` and `--dry-run` cannot be used together. + +### Add Endpoint Compared to Allow and Deny + +`--add-endpoint` works at the endpoint and rule level. It creates a new `network_policies` entry when needed, or merges into an existing rule that already covers the same host and port. Use it when you are defining where traffic may go and which binaries may send it. + +`--add-allow` and `--add-deny` work at the REST request level. They do not create binaries, and they do not create a new endpoint. They modify an existing endpoint that already has `protocol: rest`. + +This is the practical difference: + +- Use `--add-endpoint` to say "allow this binary to reach `api.github.com:443`." +- Use `--add-allow` to say "for that existing REST endpoint, also allow `POST /repos/*/issues`." +- Use `--add-deny` to say "for that existing REST endpoint, explicitly deny `POST /admin/**`." + +In the first pass of this feature: + +- `--add-allow` and `--add-deny` only work on `protocol: rest` endpoints. +- `--add-deny` requires the endpoint to already have an allow base, either an `access` preset or explicit allow `rules`. +- `protocol: sql` is not a practical incremental workflow today. OpenShell does not do full SQL parsing, and SQL enforcement is not meaningfully supported yet. + +### Endpoint Specs + +`--add-endpoint` uses this format: + +```text +host:port[:access[:protocol[:enforcement]]] +``` + +Each segment has a fixed meaning: + +| Segment | Required | Meaning | +|---|---|---| +| `host` | Yes | Destination hostname. | +| `port` | Yes | Destination port, `1` through `65535`. | +| `access` | No | Access preset for REST endpoints: `read-only`, `read-write`, or `full`. | +| `protocol` | No | L7 inspection mode: `rest` or `sql`. In practice, incremental updates are designed around `rest`. `sql` is audit-only and not a recommended workflow today. | +| `enforcement` | No | Enforcement mode for inspected traffic: `enforce` or `audit`. | + +Examples: + +| Example | Meaning | +|---|---| +| `pypi.org:443` | Add a plain L4 endpoint. The proxy allows the TCP stream and does not inspect HTTP requests. | +| `api.github.com:443:read-only:rest:enforce` | Add a REST endpoint with the `read-only` preset expanded by the policy engine into GET, HEAD, and OPTIONS access. | + +If you set `protocol: rest`, you also need an allow shape. With incremental updates, that means you should provide an `access` preset on `--add-endpoint`, then use `--add-allow` or `--add-deny` to refine it later. + +For example: + +- `api.github.com:443:read-only:rest` is valid. +- `api.github.com:443::rest` is invalid. It does not mean "allow all traffic." A REST endpoint with `protocol` but no `access` or `rules` is rejected when the policy loads. + +When you pass multiple `--add-endpoint` flags in one command, every `--binary` value applies to every added endpoint in that command. If different endpoints need different binaries, use separate `policy update` commands. + +If you do not pass `--rule-name`, OpenShell generates one from the host and port, such as `allow_api_github_com_443`. + +### REST Rule Specs + +`--add-allow` and `--add-deny` use this format: + +```text +host:port:METHOD:path_glob +``` + +This string identifies an existing REST endpoint and the request pattern you want to add. + +In shell commands, quote the full `SPEC` when it contains `*` or `**` so your shell passes it literally instead of expanding it as a local file glob. + +| Segment | Meaning | +|---|---| +| `host` | Existing endpoint host. | +| `port` | Existing endpoint port. | +| `METHOD` | HTTP method. The CLI normalizes it to uppercase. | +| `path_glob` | URL path glob. It must start with `/`, or be `**`, or start with `**/`. | + +This example: + +```text +api.github.com:443:POST:/repos/*/issues +``` + +means: + +- match the endpoint `api.github.com:443`. +- match HTTP method `POST`. +- match paths like `/repos/acme/issues`. +- do not match deeper paths like `/repos/acme/project/issues/123` because `*` matches one path segment. + +Path globs follow the same semantics as YAML allow and deny rules: + +- `*` matches one path segment. +- `**` matches any number of segments. +- `/repos/*/issues` matches one repository owner or name segment in the middle. +- `/repos/**` matches everything under `/repos/`. + +The rule-level commands only modify method and path constraints. They do not change binaries, hostnames, ports, or protocol settings. + +### Common Workflows + +Use these patterns as starting points when you decide whether to update an endpoint or append REST rules. + +#### Add a new L4 endpoint + +Use `--add-endpoint` when you need a new host and port and do not need REST inspection. + +```shell +openshell policy update demo \ + --add-endpoint pypi.org:443 \ + --add-endpoint files.pythonhosted.org:443 \ + --binary /usr/bin/pip \ + --binary /usr/local/bin/uv \ + --wait +``` + +This creates or merges endpoint entries and binds them to the listed binaries. It does not create per-path REST rules. + +#### Create a REST endpoint with a base allow set + +Use `--add-endpoint` first when the endpoint does not exist yet. + +```shell +openshell policy update demo \ + --add-endpoint api.github.com:443:read-only:rest:enforce \ + --binary /usr/bin/gh \ + --wait +``` + +This creates a REST endpoint and sets its base allow behavior through the `read-only` access preset. + +#### Add one more REST allow rule + +Use `--add-allow` after the REST endpoint already exists. + +```shell +openshell policy update demo \ + --add-allow 'api.github.com:443:POST:/repos/*/issues' \ + --wait +``` + +This keeps the existing endpoint definition and appends one new allow rule. It does not add binaries or change the endpoint host and port. + +#### Add a REST deny rule under an allowed endpoint + +Use `--add-deny` when you want to carve out a blocked subtree under an existing REST endpoint. + +```shell +openshell policy update demo \ + --add-deny 'api.github.com:443:POST:/admin/**' \ + --wait +``` + +This adds a deny rule to the existing REST endpoint. The endpoint must already have an allow base. + +#### Remove one endpoint or rule + +Use `--remove-endpoint` to remove one host and port pair, or `--remove-rule` to delete the whole named rule. + +```shell +openshell policy update demo --remove-endpoint pypi.org:443 --wait +openshell policy update demo --remove-rule github_repos --wait +``` + +If the target endpoint is part of a multi-port endpoint, `--remove-endpoint` removes only the specified port and keeps the rest. + +### Merge Semantics + +OpenShell applies all update flags from one `openshell policy update` command as one merge batch. The gateway validates the full merged result and persists at most one new policy revision. + +This means: + +- one command is atomic at the revision level. +- multiple flags in one command succeed or fail together. +- concurrent writers do not partially interleave one batch with another. + +When two updates race, the gateway uses optimistic retry. It fetches the latest revision, reapplies the full batch, validates the result again, and retries the write. This preserves the intent of each individual command while still allowing concurrent sandbox policy updates. + +### Preview and Validation + +Use `--dry-run` when you want to inspect the merged YAML before you send it to the gateway. + +```shell +openshell policy update demo \ + --add-allow 'api.github.com:443:GET:/repos/**' \ + --dry-run +``` + +The CLI validates the argument shapes before it sends the request. The gateway then validates the merged policy against the current live policy and returns clear errors when: + +- a required segment is missing. +- a port is outside `1` through `65535`. +- `--add-allow` or `--add-deny` points at an endpoint that does not exist. +- `--add-allow` or `--add-deny` targets a non-REST endpoint. +- `--add-deny` targets an endpoint that has no base allow set. + ## Global Policy Override Use a global policy when you want one policy payload to apply to every sandbox. @@ -178,9 +417,15 @@ When triaging denied requests, check: Then push the updated policy as described above. +For small changes, prefer `openshell policy update` over rewriting the full YAML: + +```shell +openshell policy update --add-allow 'api.github.com:443:GET:/repos/**' --wait +``` + ## Examples -Add these blocks to the `network_policies` section of your sandbox policy. Apply with `openshell policy set --policy --wait`. +Add these blocks to the `network_policies` section of your sandbox policy. Apply with `openshell policy update` for incremental additions or `openshell policy set --policy --wait` for full replacement. Use **Simple endpoint** for host-level allowlists and **Granular rules** for method/path control. diff --git a/docs/security/best-practices.mdx b/docs/security/best-practices.mdx index 52fc82131c..c28d6a68ff 100644 --- a/docs/security/best-practices.mdx +++ b/docs/security/best-practices.mdx @@ -24,11 +24,11 @@ If you use [NemoClaw](https://github.com/NVIDIA/NemoClaw) to run OpenClaw assist OpenShell applies security controls at two enforcement points. OpenShell locks static controls at sandbox creation and requires destroying and recreating the sandbox to change them. -You can update dynamic controls on a running sandbox with `openshell policy set`. +You can update dynamic controls on a running sandbox with `openshell policy update` or `openshell policy set`. | Layer | What it protects | Enforcement point | Changeable at runtime | | --- | --- | --- | --- | -| Network | Unauthorized outbound connections and data exfiltration. | CONNECT proxy + OPA policy engine | Yes. Use `openshell policy set` or operator approval in the TUI. | +| Network | Unauthorized outbound connections and data exfiltration. | CONNECT proxy + OPA policy engine | Yes. Use `openshell policy update`, `openshell policy set`, or operator approval in the TUI. | | Filesystem | System binary tampering, credential theft, config manipulation. | Landlock LSM (kernel level) | No. Requires sandbox re-creation. | | Process | Privilege escalation, fork bombs, dangerous syscalls. | Seccomp BPF + privilege drop (`setuid`/`setgid`) | No. Requires sandbox re-creation. | | Inference | Credential exposure, unauthorized model access. | Proxy intercept of `inference.local` | Yes. Use `openshell inference set`. | @@ -46,7 +46,7 @@ If no `network_policies` entry matches the destination host, port, and calling b | Aspect | Detail | |---|---| | Default | All egress denied. Only endpoints listed in `network_policies` can receive traffic. | -| What you can change | Add entries to `network_policies` in the policy YAML. Apply statically at creation (`--policy`) or dynamically (`openshell policy set`). | +| What you can change | Add entries to `network_policies` in the policy YAML. Apply statically at creation (`--policy`) or dynamically (`openshell policy update` for incremental changes, `openshell policy set` for full replacement). | | Risk if relaxed | Each allowed endpoint is a potential data exfiltration path. The agent can send workspace content, credentials, or conversation history to any reachable host. | | Recommendation | Add only endpoints the agent needs for its task. Start with a minimal policy and use denied-request logs (`openshell logs --source sandbox`) to identify missing endpoints. | diff --git a/proto/openshell.proto b/proto/openshell.proto index 0ee1e89041..b863e62517 100644 --- a/proto/openshell.proto +++ b/proto/openshell.proto @@ -574,6 +574,51 @@ message UpdateConfigRequest { bool delete_setting = 5; // Apply mutation at gateway-global scope. bool global = 6; + // Batched incremental policy merge operations. Sandbox-scoped only. + repeated PolicyMergeOperation merge_operations = 7; +} + +message PolicyMergeOperation { + oneof operation { + AddNetworkRule add_rule = 1; + RemoveNetworkEndpoint remove_endpoint = 2; + RemoveNetworkRule remove_rule = 3; + AddDenyRules add_deny_rules = 4; + AddAllowRules add_allow_rules = 5; + RemoveNetworkBinary remove_binary = 6; + } +} + +message AddNetworkRule { + string rule_name = 1; + openshell.sandbox.v1.NetworkPolicyRule rule = 2; +} + +message RemoveNetworkEndpoint { + string rule_name = 1; + string host = 2; + uint32 port = 3; +} + +message RemoveNetworkRule { + string rule_name = 1; +} + +message AddDenyRules { + string host = 1; + uint32 port = 2; + repeated openshell.sandbox.v1.L7DenyRule deny_rules = 3; +} + +message AddAllowRules { + string host = 1; + uint32 port = 2; + repeated openshell.sandbox.v1.L7Rule rules = 3; +} + +message RemoveNetworkBinary { + string rule_name = 1; + string binary_path = 2; } // Update sandbox policy response. diff --git a/tasks/scripts/cluster-deploy-fast.sh b/tasks/scripts/cluster-deploy-fast.sh index c38259288c..d9359cf4f8 100755 --- a/tasks/scripts/cluster-deploy-fast.sh +++ b/tasks/scripts/cluster-deploy-fast.sh @@ -152,7 +152,7 @@ matches_gateway() { deploy/docker/Dockerfile.images|tasks/scripts/docker-build-image.sh) return 0 ;; - crates/openshell-core/*|crates/openshell-driver-kubernetes/*|crates/openshell-policy/*|crates/openshell-providers/*) + crates/openshell-core/*|crates/openshell-driver-kubernetes/*|crates/openshell-ocsf/*|crates/openshell-policy/*|crates/openshell-providers/*) return 0 ;; crates/openshell-router/*|crates/openshell-server/*) @@ -209,7 +209,7 @@ compute_fingerprint() { local committed_trees="" case "${component}" in gateway) - committed_trees=$(git ls-tree HEAD Cargo.toml Cargo.lock proto/ deploy/docker/cross-build.sh deploy/docker/Dockerfile.images tasks/scripts/docker-build-image.sh crates/openshell-core/ crates/openshell-driver-kubernetes/ crates/openshell-policy/ crates/openshell-providers/ crates/openshell-router/ crates/openshell-server/ 2>/dev/null || true) + committed_trees=$(git ls-tree HEAD Cargo.toml Cargo.lock proto/ deploy/docker/cross-build.sh deploy/docker/Dockerfile.images tasks/scripts/docker-build-image.sh crates/openshell-core/ crates/openshell-driver-kubernetes/ crates/openshell-ocsf/ crates/openshell-policy/ crates/openshell-providers/ crates/openshell-router/ crates/openshell-server/ 2>/dev/null || true) ;; supervisor) committed_trees=$(git ls-tree HEAD Cargo.toml Cargo.lock proto/ deploy/docker/cross-build.sh deploy/docker/Dockerfile.images tasks/scripts/docker-build-image.sh crates/openshell-core/ crates/openshell-policy/ crates/openshell-router/ crates/openshell-sandbox/ 2>/dev/null || true) From 7a0a3d0cced31e290775b81165cb9a7a0668fb5f Mon Sep 17 00:00:00 2001 From: "John T. Myers" <9696606+johntmyers@users.noreply.github.com> Date: Mon, 20 Apr 2026 11:38:20 -0700 Subject: [PATCH 014/142] fix(cli,tui): escape and validate SSH session response fields (#876) The CLI and TUI build an SSH ProxyCommand string by interpolating fields from CreateSshSessionResponse into a shell command. Three fields - sandbox_id, token, and the assembled gateway_url (composed from gateway_scheme, gateway_host, gateway_port, connect_path) - were not passed through shell_escape, even though exe_command and gateway_name were. The resulting string is handed to `ssh -o ProxyCommand=...` which OpenSSH routes through `/bin/sh -c`, so a hostile or compromised gateway could inject shell metacharacters and execute arbitrary commands under the developer's workstation account. - Add build_proxy_command in openshell-core::forward that wraps every interpolated value in shell_escape. Use it at all four ProxyCommand sites: openshell-cli/src/ssh.rs (ssh_session_config) and the three TUI sites in openshell-tui/src/lib.rs (shell connect, sandbox exec, port-forward reconciliation). - Add validate_ssh_session_response in openshell-core::forward, called at each site immediately after response.into_inner(). Enforces a conservative charset per field (sandbox_id, token, gateway_host, gateway_scheme, gateway_port, connect_path, host_key_fingerprint) so malformed responses fail loudly at the gRPC trust boundary before shell escaping is attempted. Belt-and-suspenders with the escape. - Harden render_ssh_config so `gateway` and `name` are shell-escaped even though validate_gateway_name currently gates `gateway`. - Document the charset contract on CreateSshSessionResponse in proto/openshell.proto: servers must uphold these rules and clients reject responses that violate them. - Add regression unit tests covering adversarial values in every field and a build_proxy_command test asserting no shell metacharacter remains outside single-quoted regions. Tracks OS-100. --- crates/openshell-cli/src/ssh.rs | 25 +- crates/openshell-core/src/forward.rs | 378 +++++++++++++++++++++++++++ crates/openshell-tui/src/lib.rs | 49 ++-- proto/openshell.proto | 26 +- 4 files changed, 446 insertions(+), 32 deletions(-) diff --git a/crates/openshell-cli/src/ssh.rs b/crates/openshell-cli/src/ssh.rs index ebcbbeb4f6..2bee1b7bc0 100644 --- a/crates/openshell-cli/src/ssh.rs +++ b/crates/openshell-cli/src/ssh.rs @@ -8,7 +8,8 @@ use miette::{IntoDiagnostic, Result, WrapErr}; #[cfg(unix)] use nix::sys::signal::{SaFlags, SigAction, SigHandler, SigSet, Signal, sigaction}; use openshell_core::forward::{ - find_ssh_forward_pid, resolve_ssh_gateway, shell_escape, write_forward_pid, + build_proxy_command, find_ssh_forward_pid, resolve_ssh_gateway, shell_escape, + validate_ssh_session_response, write_forward_pid, }; use openshell_core::proto::{CreateSshSessionRequest, GetSandboxRequest}; use owo_colors::OwoColorize; @@ -86,11 +87,13 @@ async fn ssh_session_config( .await .into_diagnostic()?; let session = response.into_inner(); + validate_ssh_session_response(&session) + .map_err(|err| miette::miette!("gateway returned invalid SSH session response: {err}"))?; let exe = std::env::current_exe() .into_diagnostic() .wrap_err("failed to resolve OpenShell executable")?; - let exe_command = shell_escape(&exe.to_string_lossy()); + let exe_command = exe.to_string_lossy().into_owned(); // When using Cloudflare bearer auth, the SSH CONNECT must go through the // external tunnel endpoint (the cluster URL), not the server's internal @@ -114,12 +117,12 @@ async fn ssh_session_config( let gateway_name = tls .gateway_name() .ok_or_else(|| miette::miette!("gateway name is required to build SSH proxy command"))?; - let proxy_command = format!( - "{exe_command} ssh-proxy --gateway {} --sandbox-id {} --token {} --gateway-name {}", - gateway_url, - session.sandbox_id, - session.token, - shell_escape(gateway_name), + let proxy_command = build_proxy_command( + &exe_command, + &gateway_url, + &session.sandbox_id, + &session.token, + gateway_name, ); Ok(SshSessionConfig { @@ -867,7 +870,11 @@ fn render_ssh_config(gateway: &str, name: &str) -> String { let exe = std::env::current_exe().expect("failed to resolve OpenShell executable"); let exe = shell_escape(&exe.to_string_lossy()); - let proxy_cmd = format!("{exe} ssh-proxy --gateway-name {gateway} --name {name}"); + let proxy_cmd = format!( + "{exe} ssh-proxy --gateway-name {} --name {}", + shell_escape(gateway), + shell_escape(name), + ); let host_alias = host_alias(name); format!( "Host {host_alias}\n User sandbox\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n GlobalKnownHostsFile /dev/null\n LogLevel ERROR\n ProxyCommand {proxy_cmd}\n" diff --git a/crates/openshell-core/src/forward.rs b/crates/openshell-core/src/forward.rs index c7b63fefb7..e6c7d2f7cc 100644 --- a/crates/openshell-core/src/forward.rs +++ b/crates/openshell-core/src/forward.rs @@ -487,6 +487,188 @@ pub fn shell_escape(value: &str) -> String { format!("'{escaped}'") } +/// Build the SSH `ProxyCommand` string used to tunnel to a sandbox. +/// +/// Every interpolated argument is shell-escaped so that server-supplied values +/// (gateway URL, sandbox id, token, gateway name) cannot inject shell +/// metacharacters into the command that OpenSSH executes via `/bin/sh -c`. +pub fn build_proxy_command( + exe: &str, + gateway_url: &str, + sandbox_id: &str, + token: &str, + gateway_name: &str, +) -> String { + format!( + "{} ssh-proxy --gateway {} --sandbox-id {} --token {} --gateway-name {}", + shell_escape(exe), + shell_escape(gateway_url), + shell_escape(sandbox_id), + shell_escape(token), + shell_escape(gateway_name), + ) +} + +/// Error returned when a `CreateSshSessionResponse` fails validation. +/// +/// The response fields flow into a `ProxyCommand` string executed by +/// `/bin/sh -c`; any deviation from the documented charset is rejected at the +/// gRPC trust boundary before escaping is attempted. +#[derive(Debug, thiserror::Error)] +pub enum SshSessionResponseError { + #[error("{field} is empty")] + Empty { field: &'static str }, + #[error("{field} exceeds maximum length of {max} bytes")] + TooLong { field: &'static str, max: usize }, + #[error("{field} contains invalid characters")] + InvalidChars { field: &'static str }, + #[error("gateway_scheme must be 'http' or 'https'")] + InvalidScheme, + #[error("gateway_port must be in range 1..=65535")] + InvalidPort, + #[error("connect_path must start with '/'")] + ConnectPathNotAbsolute, +} + +const MAX_SANDBOX_ID_LEN: usize = 128; +const MAX_TOKEN_LEN: usize = 4096; +const MAX_GATEWAY_HOST_LEN: usize = 253; +const MAX_CONNECT_PATH_LEN: usize = 2048; +const MAX_FINGERPRINT_LEN: usize = 256; + +fn is_sandbox_id_byte(b: u8) -> bool { + b.is_ascii_alphanumeric() || matches!(b, b'.' | b'-' | b'_') +} + +fn is_token_byte(b: u8) -> bool { + // URL-safe base64 + common token charset. No shell metacharacters, no + // whitespace, no control bytes. + b.is_ascii_alphanumeric() || matches!(b, b'.' | b'-' | b'_' | b'~' | b'+' | b'/' | b'=') +} + +fn is_gateway_host_byte(b: u8) -> bool { + // DNS hostname (alphanumeric + `.-`), IPv4, or bracketed IPv6 (`[::1]`). + // Rejects Unicode — callers must Punycode-encode IDN hosts before emitting. + b.is_ascii_alphanumeric() || matches!(b, b'.' | b'-' | b':' | b'[' | b']') +} + +fn is_connect_path_byte(b: u8) -> bool { + // RFC 3986 path charset (pchar) without `?`, `#`, space, backtick, or + // backslash. `%` is permitted so percent-encoded segments round-trip. + b.is_ascii_alphanumeric() + || matches!( + b, + b'-' | b'.' + | b'_' + | b'~' + | b'!' + | b'$' + | b'&' + | b'\'' + | b'(' + | b')' + | b'*' + | b'+' + | b',' + | b';' + | b'=' + | b':' + | b'@' + | b'/' + | b'%' + ) +} + +fn is_fingerprint_byte(b: u8) -> bool { + b.is_ascii_alphanumeric() || matches!(b, b':' | b'+' | b'/' | b'=' | b'-') +} + +/// Validate a `CreateSshSessionResponse` before any of its fields are used to +/// build a shell command or config file. +/// +/// This is a belt-and-suspenders pair to [`build_proxy_command`]: escaping +/// alone is sufficient to prevent injection, but rejecting malformed fields +/// at the trust boundary fails loudly before the string is assembled and +/// catches gateway bugs or tampering early. +pub fn validate_ssh_session_response( + resp: &crate::proto::CreateSshSessionResponse, +) -> std::result::Result<(), SshSessionResponseError> { + validate_field( + "sandbox_id", + &resp.sandbox_id, + MAX_SANDBOX_ID_LEN, + is_sandbox_id_byte, + )?; + validate_field("token", &resp.token, MAX_TOKEN_LEN, is_token_byte)?; + validate_field( + "gateway_host", + &resp.gateway_host, + MAX_GATEWAY_HOST_LEN, + is_gateway_host_byte, + )?; + match resp.gateway_scheme.as_str() { + "http" | "https" => {} + _ => return Err(SshSessionResponseError::InvalidScheme), + } + if resp.gateway_port == 0 || resp.gateway_port > u32::from(u16::MAX) { + return Err(SshSessionResponseError::InvalidPort); + } + if resp.connect_path.is_empty() { + return Err(SshSessionResponseError::Empty { + field: "connect_path", + }); + } + if !resp.connect_path.starts_with('/') { + return Err(SshSessionResponseError::ConnectPathNotAbsolute); + } + if resp.connect_path.len() > MAX_CONNECT_PATH_LEN { + return Err(SshSessionResponseError::TooLong { + field: "connect_path", + max: MAX_CONNECT_PATH_LEN, + }); + } + if !resp.connect_path.bytes().all(is_connect_path_byte) { + return Err(SshSessionResponseError::InvalidChars { + field: "connect_path", + }); + } + if !resp.host_key_fingerprint.is_empty() { + if resp.host_key_fingerprint.len() > MAX_FINGERPRINT_LEN { + return Err(SshSessionResponseError::TooLong { + field: "host_key_fingerprint", + max: MAX_FINGERPRINT_LEN, + }); + } + if !resp.host_key_fingerprint.bytes().all(is_fingerprint_byte) { + return Err(SshSessionResponseError::InvalidChars { + field: "host_key_fingerprint", + }); + } + } + Ok(()) +} + +fn validate_field( + name: &'static str, + value: &str, + max_len: usize, + byte_ok: fn(u8) -> bool, +) -> std::result::Result<(), SshSessionResponseError> { + if value.is_empty() { + return Err(SshSessionResponseError::Empty { field: name }); + } + if value.len() > max_len { + return Err(SshSessionResponseError::TooLong { + field: name, + max: max_len, + }); + } + if !value.bytes().all(byte_ok) { + return Err(SshSessionResponseError::InvalidChars { field: name }); + } + Ok(()) +} + /// Build notes string for a sandbox based on active forwards. /// /// Returns a string like `fwd:8080,3000` or an empty string if no forwards @@ -569,6 +751,202 @@ mod tests { assert_eq!(shell_escape("it's"), "'it'\"'\"'s'"); } + fn valid_session_response() -> crate::proto::CreateSshSessionResponse { + crate::proto::CreateSshSessionResponse { + sandbox_id: "sb-1234".to_string(), + token: "abcDEF-123_456.789".to_string(), + gateway_scheme: "https".to_string(), + gateway_host: "gateway.example.com".to_string(), + gateway_port: 443, + connect_path: "/connect/ssh".to_string(), + host_key_fingerprint: String::new(), + expires_at_ms: 0, + } + } + + #[test] + fn validate_ssh_session_response_accepts_realistic_response() { + assert!(validate_ssh_session_response(&valid_session_response()).is_ok()); + } + + #[test] + fn validate_ssh_session_response_accepts_bracketed_ipv6_host() { + let mut r = valid_session_response(); + r.gateway_host = "[::1]".to_string(); + assert!(validate_ssh_session_response(&r).is_ok()); + } + + #[test] + fn validate_ssh_session_response_accepts_optional_fingerprint() { + let mut r = valid_session_response(); + r.host_key_fingerprint = "SHA256:abcd+/=".to_string(); + assert!(validate_ssh_session_response(&r).is_ok()); + } + + #[test] + fn validate_ssh_session_response_rejects_empty_sandbox_id() { + let mut r = valid_session_response(); + r.sandbox_id.clear(); + assert!(matches!( + validate_ssh_session_response(&r), + Err(SshSessionResponseError::Empty { + field: "sandbox_id" + }) + )); + } + + #[test] + fn validate_ssh_session_response_rejects_shell_metachars_in_sandbox_id() { + for bad in ["a;b", "a b", "a$(id)", "a`id`", "a|b", "a&b", "a\nb"] { + let mut r = valid_session_response(); + r.sandbox_id = bad.to_string(); + assert!( + validate_ssh_session_response(&r).is_err(), + "expected reject for sandbox_id={bad:?}" + ); + } + } + + #[test] + fn validate_ssh_session_response_rejects_shell_metachars_in_token() { + for bad in ["$(id)", "`id`", "a;b", "a b", "a\tb", "a\0b"] { + let mut r = valid_session_response(); + r.token = bad.to_string(); + assert!( + validate_ssh_session_response(&r).is_err(), + "expected reject for token={bad:?}" + ); + } + } + + #[test] + fn validate_ssh_session_response_rejects_invalid_gateway_host() { + for bad in ["evil; cmd", "evil host", "ev$(id)il", "ev\nil", "evil/x"] { + let mut r = valid_session_response(); + r.gateway_host = bad.to_string(); + assert!( + validate_ssh_session_response(&r).is_err(), + "expected reject for gateway_host={bad:?}" + ); + } + } + + #[test] + fn validate_ssh_session_response_rejects_unknown_scheme() { + for bad in ["javascript", "file", "", "HTTPS", "ftp"] { + let mut r = valid_session_response(); + r.gateway_scheme = bad.to_string(); + assert!( + matches!( + validate_ssh_session_response(&r), + Err(SshSessionResponseError::InvalidScheme) + ), + "expected InvalidScheme for scheme={bad:?}" + ); + } + } + + #[test] + fn validate_ssh_session_response_rejects_out_of_range_port() { + for bad in [0u32, 65_536, 100_000] { + let mut r = valid_session_response(); + r.gateway_port = bad; + assert!(matches!( + validate_ssh_session_response(&r), + Err(SshSessionResponseError::InvalidPort) + )); + } + } + + #[test] + fn validate_ssh_session_response_rejects_connect_path_without_leading_slash() { + let mut r = valid_session_response(); + r.connect_path = "connect/ssh".to_string(); + assert!(matches!( + validate_ssh_session_response(&r), + Err(SshSessionResponseError::ConnectPathNotAbsolute) + )); + } + + #[test] + fn validate_ssh_session_response_rejects_injected_connect_path() { + // `$`, `(`, `)` are valid RFC 3986 sub-delims (pchar) so the validator + // permits them; shell_escape is the second defensive layer. The + // following characters are rejected at the validator boundary because + // they are either unambiguously hostile in a shell context or invalid + // per RFC 3986 in the path component. + for bad in ["/x`id`y", "/x y", "/x\nb", "/x\\b", "/x?q=1", "/x#frag"] { + let mut r = valid_session_response(); + r.connect_path = bad.to_string(); + assert!( + validate_ssh_session_response(&r).is_err(), + "expected reject for connect_path={bad:?}" + ); + } + } + + #[test] + fn build_proxy_command_escapes_shell_metacharacters() { + // Attacker-controlled values in every escapable position. + let cmd = build_proxy_command( + "/usr/local/bin/openshell", + "https://gw:443/connect", + "x$(touch /tmp/pwn)x", + "tok`id`", + "gw-name", + ); + + // The `$` / backtick must only appear inside single-quoted regions. + // A simple grep-based check: split on single-quoted runs and assert + // no shell metacharacter remains in the unquoted remainder. + assert!(!outside_single_quotes(&cmd).contains('$')); + assert!(!outside_single_quotes(&cmd).contains('`')); + assert!(!outside_single_quotes(&cmd).contains('|')); + assert!(!outside_single_quotes(&cmd).contains(';')); + assert!(!outside_single_quotes(&cmd).contains('&')); + assert!(!outside_single_quotes(&cmd).contains('\n')); + } + + #[test] + fn build_proxy_command_empty_values_quote_rather_than_vanish() { + // An empty value must become `''` rather than disappearing — otherwise + // downstream argv splitting would misalign. + let cmd = build_proxy_command("exe", "gw", "", "tok", "name"); + assert!(cmd.contains("--sandbox-id ''")); + } + + #[test] + fn build_proxy_command_safe_values_pass_through_unquoted() { + let cmd = build_proxy_command( + "/usr/local/bin/openshell", + "gw", + "sb-123", + "tok.456", + "name_1", + ); + assert_eq!( + cmd, + "/usr/local/bin/openshell ssh-proxy --gateway gw --sandbox-id sb-123 --token tok.456 --gateway-name name_1" + ); + } + + /// Helper: return the concatenation of characters that appear outside + /// POSIX single-quoted runs. Used by the metacharacter assertions above. + fn outside_single_quotes(s: &str) -> String { + let mut out = String::new(); + let mut inside = false; + for c in s.chars() { + if c == '\'' { + inside = !inside; + continue; + } + if !inside { + out.push(c); + } + } + out + } + #[test] fn build_sandbox_notes_with_forwards() { let forwards = vec![ diff --git a/crates/openshell-tui/src/lib.rs b/crates/openshell-tui/src/lib.rs index 63cfb79d6d..affdbc224f 100644 --- a/crates/openshell-tui/src/lib.rs +++ b/crates/openshell-tui/src/lib.rs @@ -834,6 +834,10 @@ async fn handle_shell_connect( } } }; + if let Err(err) = validate_ssh_session_response(&session) { + app.status_text = format!("gateway returned invalid SSH session response: {err}"); + return; + } // Step 3: Resolve gateway address (handle loopback override). #[allow(clippy::cast_possible_truncation)] @@ -853,11 +857,12 @@ async fn handle_shell_connect( return; } }; - let exe_str = shell_escape(&exe.to_string_lossy()); - let gateway = shell_escape(&app.gateway_name); - let proxy_command = format!( - "{exe_str} ssh-proxy --gateway {gateway_url} --sandbox-id {} --token {} --gateway-name {gateway}", - session.sandbox_id, session.token, + let proxy_command = build_proxy_command( + &exe.to_string_lossy(), + &gateway_url, + &session.sandbox_id, + &session.token, + &app.gateway_name, ); // Step 5: Build the SSH command. let mut command = std::process::Command::new("ssh"); @@ -977,6 +982,10 @@ async fn handle_exec_command( } } }; + if let Err(err) = validate_ssh_session_response(&session) { + app.status_text = format!("exec: gateway returned invalid SSH session response: {err}"); + return; + } // Step 2: Resolve gateway and build ProxyCommand (same as handle_shell_connect). #[allow(clippy::cast_possible_truncation)] @@ -995,11 +1004,12 @@ async fn handle_exec_command( return; } }; - let exe_str = shell_escape(&exe.to_string_lossy()); - let gateway = shell_escape(&app.gateway_name); - let proxy_command = format!( - "{exe_str} ssh-proxy --gateway {gateway_url} --sandbox-id {} --token {} --gateway-name {gateway}", - session.sandbox_id, session.token, + let proxy_command = build_proxy_command( + &exe.to_string_lossy(), + &gateway_url, + &session.sandbox_id, + &session.token, + &app.gateway_name, ); // Step 3: Build SSH command — same flags as handle_shell_connect but with @@ -1073,7 +1083,9 @@ async fn handle_exec_command( } // SSH utility functions are shared via openshell_core::forward. -use openshell_core::forward::{resolve_ssh_gateway, shell_escape}; +use openshell_core::forward::{ + build_proxy_command, resolve_ssh_gateway, shell_escape, validate_ssh_session_response, +}; /// Convert a `SandboxPolicy` proto into styled ratatui lines for the policy viewer. fn render_policy_lines( @@ -1400,6 +1412,10 @@ async fn start_port_forwards( } } }; + if let Err(err) = validate_ssh_session_response(&session) { + tracing::warn!("gateway returned invalid SSH session response for forwards: {err}"); + return; + } // Resolve gateway address. #[allow(clippy::cast_possible_truncation)] @@ -1419,11 +1435,12 @@ async fn start_port_forwards( return; } }; - let exe_str = shell_escape(&exe.to_string_lossy()); - let gateway = shell_escape(gateway_name); - let proxy_command = format!( - "{exe_str} ssh-proxy --gateway {gateway_url} --sandbox-id {} --token {} --gateway-name {gateway}", - session.sandbox_id, session.token, + let proxy_command = build_proxy_command( + &exe.to_string_lossy(), + &gateway_url, + &session.sandbox_id, + &session.token, + gateway_name, ); // Start a forward for each spec. diff --git a/proto/openshell.proto b/proto/openshell.proto index b863e62517..895b0d1aa9 100644 --- a/proto/openshell.proto +++ b/proto/openshell.proto @@ -315,26 +315,38 @@ message CreateSshSessionRequest { } // Create SSH session response. +// +// Fields are interpolated into an SSH `ProxyCommand` string that OpenSSH +// executes through `/bin/sh -c` on the caller's workstation. Servers MUST +// uphold the charset contract below; clients MUST reject responses that +// violate it. The client's own escaping provides defense-in-depth, but +// narrow charsets close injection vectors at the trust boundary. message CreateSshSessionResponse { - // Sandbox id. + // Sandbox id. [A-Za-z0-9._-]{1,128}. string sandbox_id = 1; - // Session token for the gateway tunnel. + // Session token for the gateway tunnel. URL-safe ASCII + // ([A-Za-z0-9._~+/=-]) up to 4096 bytes. No shell metacharacters or + // whitespace. string token = 2; - // Gateway host for SSH proxy connection. + // Gateway host for SSH proxy connection. IPv4 address, bracketed IPv6 + // address, or DNS hostname (Punycode-encoded for IDN). Alphanumeric plus + // `.-:[]` only, up to 253 bytes. string gateway_host = 3; - // Gateway port for SSH proxy connection. + // Gateway port for SSH proxy connection. Must be in range 1..=65535. uint32 gateway_port = 4; - // Gateway scheme (http or https). + // Gateway scheme. Must be exactly "http" or "https". string gateway_scheme = 5; - // HTTP path for the CONNECT/upgrade endpoint. + // HTTP path for the CONNECT/upgrade endpoint. Must begin with `/`. RFC + // 3986 path charset only ([A-Za-z0-9._~!$&'()*+,;=:@/-] plus %HH). + // Must not contain `?`, `#`, whitespace, backtick, or backslash. string connect_path = 6; - // Optional host key fingerprint. + // Optional host key fingerprint. If non-empty, [A-Za-z0-9:+/=-] only. string host_key_fingerprint = 7; // Expiry timestamp in milliseconds since epoch. 0 means no expiry. From 8a813aba493e21c352bd316f7409ac6168ec6117 Mon Sep 17 00:00:00 2001 From: "John T. Myers" <9696606+johntmyers@users.noreply.github.com> Date: Mon, 20 Apr 2026 13:09:23 -0700 Subject: [PATCH 015/142] fix(sandbox): apply supervisor seccomp prelude (#891) * fix(sandbox): apply supervisor seccomp prelude Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> * fix(sandbox): delay supervisor prelude until bootstrap --------- Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- architecture/sandbox-custom-containers.md | 3 +- crates/openshell-sandbox/src/lib.rs | 6 + crates/openshell-sandbox/src/main.rs | 230 +++++++++--------- .../src/sandbox/linux/mod.rs | 5 + .../src/sandbox/linux/seccomp.rs | 196 ++++++++++++--- crates/openshell-sandbox/src/sandbox/mod.rs | 14 ++ docs/security/best-practices.mdx | 21 +- 7 files changed, 326 insertions(+), 149 deletions(-) diff --git a/architecture/sandbox-custom-containers.md b/architecture/sandbox-custom-containers.md index e44de29dd2..cbdc9a6661 100644 --- a/architecture/sandbox-custom-containers.md +++ b/architecture/sandbox-custom-containers.md @@ -98,6 +98,7 @@ The `openshell-sandbox` supervisor adapts to arbitrary environments: - **Log file fallback**: Attempts to open `/var/log/openshell.log` for append; silently falls back to stdout-only logging if the path is not writable. - **Command resolution**: Executes the command from CLI args, then the `OPENSHELL_SANDBOX_COMMAND` env var (set to `sleep infinity` by the server), then `/bin/bash` as a last resort. +- **Startup seccomp prelude**: Before parsing CLI args or starting the async runtime, the supervisor sets `PR_SET_NO_NEW_PRIVS` and installs a narrow seccomp filter that blocks mount/remount, the new mount API syscalls, module loading, kexec, `bpf`, `perf_event_open`, and `userfaultfd`. This closes the privileged remount window while still leaving required child-setup syscalls such as `setns` available. - **Network namespace**: Requires successful namespace creation for proxy isolation; startup fails in proxy mode if required capabilities (`CAP_NET_ADMIN`, `CAP_SYS_ADMIN`) or `iproute2` are unavailable. If the `iptables` package is present, the supervisor installs OUTPUT chain rules (LOG + REJECT) inside the namespace to provide fast-fail behavior (immediate `ECONNREFUSED` instead of a 30-second timeout) and diagnostic logging when processes attempt direct connections that bypass the HTTP CONNECT proxy. If `iptables` is absent, the supervisor logs a warning and continues — core network isolation still works via routing. ## Design Decisions @@ -109,7 +110,7 @@ The `openshell-sandbox` supervisor adapts to arbitrary environments: | Auto build+push for Dockerfiles | Eliminates the two-step `image push` + `create` workflow for local development | | `OPENSHELL_COMMUNITY_REGISTRY` env var | Allows organizations to host their own community sandbox registry | | hostPath side-load | Supervisor binary lives on the node filesystem — no init container, no emptyDir, no extra image pull. Faster pod startup. | -| Read-only mount in agent | Supervisor binary cannot be tampered with by the workload | +| Read-only mount in agent | The supervisor binary is mounted read-only, and the startup seccomp prelude blocks the remount syscalls that would otherwise reopen it for writes once privileged bootstrap has completed. | | Command override | Ensures `openshell-sandbox` is the entrypoint regardless of the image's default CMD | | Clear `run_as_user/group` for custom images | Prevents startup failure when the image lacks the default `sandbox` user | | Non-fatal log file init | `/var/log/openshell.log` may be unwritable in arbitrary images; falls back to stdout | diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 1fbbe90d42..cfc6c3051d 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -97,6 +97,7 @@ use crate::proxy::ProxyHandle; use crate::sandbox::linux::netns::NetworkNamespace; use crate::secrets::SecretResolver; pub use process::{ProcessHandle, ProcessStatus}; +pub use sandbox::apply_supervisor_startup_hardening; /// Default interval (seconds) for re-fetching the inference route bundle from /// the gateway in cluster mode. Override at runtime with the @@ -416,6 +417,11 @@ pub async fn run_sandbox( #[allow(clippy::no_effect_underscore_binding)] let _netns: Option<()> = None; + // Install the supervisor seccomp prelude after privileged startup helpers + // (network namespace setup, iptables probes) complete, but before the SSH + // listener and workload process are exposed. + apply_supervisor_startup_hardening()?; + // Shared PID: set after process spawn so the proxy can look up // the entrypoint process's /proc/net/tcp for identity binding. let entrypoint_pid = Arc::new(AtomicU32::new(0)); diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index a37dce0e4f..ea70c79cae 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -7,7 +7,7 @@ use std::sync::Arc; use std::sync::atomic::AtomicBool; use clap::Parser; -use miette::Result; +use miette::{IntoDiagnostic, Result}; use openshell_ocsf::{OcsfJsonlLayer, OcsfShorthandLayer}; use tracing::{info, warn}; use tracing_subscriber::EnvFilter; @@ -96,8 +96,7 @@ struct Args { health_port: u16, } -#[tokio::main] -async fn main() -> Result<()> { +fn main() -> Result<()> { let args = Args::parse(); // Try to open a rolling log file; fall back to stdout-only logging if it fails @@ -118,116 +117,125 @@ async fn main() -> Result<()> { let stdout_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)); - // Install rustls crypto provider before any TLS connections (including log push). - let _ = rustls::crypto::ring::default_provider().install_default(); - - // Set up optional log push layer (gRPC mode only). - let log_push_state = if let (Some(sandbox_id), Some(endpoint)) = - (&args.sandbox_id, &args.openshell_endpoint) - { - let (tx, handle) = - openshell_sandbox::log_push::spawn_log_push_task(endpoint.clone(), sandbox_id.clone()); - let layer = openshell_sandbox::log_push::LogPushLayer::new(sandbox_id.clone(), tx); - Some((layer, handle)) - } else { - None - }; - let push_layer = log_push_state.as_ref().map(|(layer, _)| layer.clone()); - let _log_push_handle = log_push_state.map(|(_, handle)| handle); - - // Shared flag: the sandbox poll loop toggles this when the - // `ocsf_json_enabled` setting changes. The JSONL layer checks it - // on each event and short-circuits when false. - let ocsf_enabled = Arc::new(AtomicBool::new(false)); - - // Keep guards alive for the entire process. When a guard is dropped the - // non-blocking writer flushes remaining logs. - let (_file_guard, _jsonl_guard) = if let Some((file_writer, file_guard)) = file_logging { - let file_filter = EnvFilter::new("info"); - - // OCSF JSONL file: rolling appender matching the main log file - // (daily rotation, 3 files max). Created eagerly but gated by the - // enabled flag — no JSONL is written until ocsf_json_enabled is set. - let jsonl_logging = tracing_appender::rolling::RollingFileAppender::builder() - .rotation(tracing_appender::rolling::Rotation::DAILY) - .filename_prefix("openshell-ocsf") - .filename_suffix("log") - .max_log_files(3) - .build("/var/log") - .ok() - .map(|roller| { - let (writer, guard) = tracing_appender::non_blocking(roller); - let layer = OcsfJsonlLayer::new(writer).with_enabled_flag(ocsf_enabled.clone()); - (layer, guard) - }); - let (jsonl_layer, jsonl_guard) = match jsonl_logging { - Some((layer, guard)) => (Some(layer), Some(guard)), - None => (None, None), + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .into_diagnostic()?; + + let exit_code = runtime.block_on(async move { + // Install rustls crypto provider before any TLS connections (including log push). + let _ = rustls::crypto::ring::default_provider().install_default(); + + // Set up optional log push layer (gRPC mode only). + let log_push_state = if let (Some(sandbox_id), Some(endpoint)) = + (&args.sandbox_id, &args.openshell_endpoint) + { + let (tx, handle) = openshell_sandbox::log_push::spawn_log_push_task( + endpoint.clone(), + sandbox_id.clone(), + ); + let layer = openshell_sandbox::log_push::LogPushLayer::new(sandbox_id.clone(), tx); + Some((layer, handle)) + } else { + None + }; + let push_layer = log_push_state.as_ref().map(|(layer, _)| layer.clone()); + let _log_push_handle = log_push_state.map(|(_, handle)| handle); + + // Shared flag: the sandbox poll loop toggles this when the + // `ocsf_json_enabled` setting changes. The JSONL layer checks it + // on each event and short-circuits when false. + let ocsf_enabled = Arc::new(AtomicBool::new(false)); + + // Keep guards alive for the entire process. When a guard is dropped the + // non-blocking writer flushes remaining logs. + let (_file_guard, _jsonl_guard) = if let Some((file_writer, file_guard)) = file_logging { + let file_filter = EnvFilter::new("info"); + + // OCSF JSONL file: rolling appender matching the main log file + // (daily rotation, 3 files max). Created eagerly but gated by the + // enabled flag — no JSONL is written until ocsf_json_enabled is set. + let jsonl_logging = tracing_appender::rolling::RollingFileAppender::builder() + .rotation(tracing_appender::rolling::Rotation::DAILY) + .filename_prefix("openshell-ocsf") + .filename_suffix("log") + .max_log_files(3) + .build("/var/log") + .ok() + .map(|roller| { + let (writer, guard) = tracing_appender::non_blocking(roller); + let layer = OcsfJsonlLayer::new(writer).with_enabled_flag(ocsf_enabled.clone()); + (layer, guard) + }); + let (jsonl_layer, jsonl_guard) = match jsonl_logging { + Some((layer, guard)) => (Some(layer), Some(guard)), + None => (None, None), + }; + + tracing_subscriber::registry() + .with( + OcsfShorthandLayer::new(std::io::stdout()) + .with_non_ocsf(true) + .with_filter(stdout_filter), + ) + .with( + OcsfShorthandLayer::new(file_writer) + .with_non_ocsf(true) + .with_filter(file_filter), + ) + .with(jsonl_layer.with_filter(LevelFilter::INFO)) + .with(push_layer.clone()) + .init(); + (Some(file_guard), jsonl_guard) + } else { + tracing_subscriber::registry() + .with( + OcsfShorthandLayer::new(std::io::stdout()) + .with_non_ocsf(true) + .with_filter(stdout_filter), + ) + .with(push_layer) + .init(); + // Log the warning after the subscriber is initialized + warn!("Could not open /var/log for log rotation; using stdout-only logging"); + (None, None) + }; + + // Get command - either from CLI args, environment variable, or default to /bin/bash + let command = if !args.command.is_empty() { + args.command + } else if let Ok(c) = std::env::var("OPENSHELL_SANDBOX_COMMAND") { + // Simple shell-like splitting on whitespace + c.split_whitespace().map(String::from).collect() + } else { + vec!["/bin/bash".to_string()] }; - tracing_subscriber::registry() - .with( - OcsfShorthandLayer::new(std::io::stdout()) - .with_non_ocsf(true) - .with_filter(stdout_filter), - ) - .with( - OcsfShorthandLayer::new(file_writer) - .with_non_ocsf(true) - .with_filter(file_filter), - ) - .with(jsonl_layer.with_filter(LevelFilter::INFO)) - .with(push_layer.clone()) - .init(); - (Some(file_guard), jsonl_guard) - } else { - tracing_subscriber::registry() - .with( - OcsfShorthandLayer::new(std::io::stdout()) - .with_non_ocsf(true) - .with_filter(stdout_filter), - ) - .with(push_layer) - .init(); - // Log the warning after the subscriber is initialized - warn!("Could not open /var/log for log rotation; using stdout-only logging"); - (None, None) - }; - - // Get command - either from CLI args, environment variable, or default to /bin/bash - let command = if !args.command.is_empty() { - args.command - } else if let Ok(c) = std::env::var("OPENSHELL_SANDBOX_COMMAND") { - // Simple shell-like splitting on whitespace - c.split_whitespace().map(String::from).collect() - } else { - vec!["/bin/bash".to_string()] - }; - - info!(command = ?command, "Starting sandbox"); - // Note: "Starting sandbox" stays as plain info!() since the OCSF context - // is not yet initialized at this point (run_sandbox hasn't been called). - // The shorthand layer will render it in fallback format. - - let exit_code = run_sandbox( - command, - args.workdir, - args.timeout, - args.interactive, - args.sandbox_id, - args.sandbox, - args.openshell_endpoint, - args.policy_rules, - args.policy_data, - args.ssh_listen_addr, - args.ssh_handshake_secret, - args.ssh_handshake_skew_secs, - args.health_check, - args.health_port, - args.inference_routes, - ocsf_enabled, - ) - .await?; + info!(command = ?command, "Starting sandbox"); + // Note: "Starting sandbox" stays as plain info!() since the OCSF context + // is not yet initialized at this point (run_sandbox hasn't been called). + // The shorthand layer will render it in fallback format. + + run_sandbox( + command, + args.workdir, + args.timeout, + args.interactive, + args.sandbox_id, + args.sandbox, + args.openshell_endpoint, + args.policy_rules, + args.policy_data, + args.ssh_listen_addr, + args.ssh_handshake_secret, + args.ssh_handshake_skew_secs, + args.health_check, + args.health_port, + args.inference_routes, + ocsf_enabled, + ) + .await + })?; std::process::exit(exit_code); } diff --git a/crates/openshell-sandbox/src/sandbox/linux/mod.rs b/crates/openshell-sandbox/src/sandbox/linux/mod.rs index 988aff1cab..565ce4f46c 100644 --- a/crates/openshell-sandbox/src/sandbox/linux/mod.rs +++ b/crates/openshell-sandbox/src/sandbox/linux/mod.rs @@ -43,6 +43,11 @@ pub fn enforce(prepared: PreparedSandbox) -> Result<()> { Ok(()) } +/// Apply the supervisor seccomp prelude after privileged bootstrap completes. +pub fn apply_supervisor_prelude() -> Result<()> { + seccomp::apply_supervisor_prelude() +} + /// Legacy single-phase apply. Kept for backward compatibility. /// New callers should use [`prepare`] + [`enforce`] for correct privilege ordering. pub fn apply(policy: &SandboxPolicy, workdir: Option<&str>) -> Result<()> { diff --git a/crates/openshell-sandbox/src/sandbox/linux/seccomp.rs b/crates/openshell-sandbox/src/sandbox/linux/seccomp.rs index 7b6f0f4127..dee96d7ced 100644 --- a/crates/openshell-sandbox/src/sandbox/linux/seccomp.rs +++ b/crates/openshell-sandbox/src/sandbox/linux/seccomp.rs @@ -16,7 +16,7 @@ use crate::policy::{NetworkMode, SandboxPolicy}; use miette::{IntoDiagnostic, Result}; use seccompiler::{ SeccompAction, SeccompCmpArgLen, SeccompCmpOp, SeccompCondition, SeccompFilter, SeccompRule, - apply_filter, + apply_filter, apply_filter_all_threads, }; use std::collections::BTreeMap; use std::convert::TryInto; @@ -25,12 +25,71 @@ use tracing::debug; /// Value of `SECCOMP_SET_MODE_FILTER` (linux/seccomp.h). const SECCOMP_SET_MODE_FILTER: u64 = 1; +/// Apply the supervisor seccomp filter across the running process. +/// +/// This runs after privileged startup helpers complete and synchronizes the +/// filter across all supervisor threads via TSYNC. It intentionally blocks +/// only the privileged escape primitives that the long-lived supervisor no +/// longer needs once bootstrap is complete. +pub fn apply_supervisor_prelude() -> Result<()> { + let filter = build_supervisor_prelude_filter()?; + set_no_new_privs()?; + apply_filter_all_threads(&filter).into_diagnostic()?; + Ok(()) +} + pub fn apply(policy: &SandboxPolicy) -> Result<()> { let allow_inet = matches!(policy.network.mode, NetworkMode::Proxy | NetworkMode::Allow); let main_filter = build_filter(allow_inet)?; let clone3_filter = build_clone3_filter()?; - // Required before applying seccomp filters. + set_no_new_privs()?; + apply_runtime_filters(&main_filter, &clone3_filter)?; + + Ok(()) +} + +fn build_filter(allow_inet: bool) -> Result { + let rules = build_filter_rules(allow_inet)?; + compile_filter(rules, SeccompAction::Errno(libc::EPERM as u32)) +} + +fn build_supervisor_prelude_filter() -> Result { + compile_filter( + build_supervisor_prelude_rules(), + SeccompAction::Errno(libc::EPERM as u32), + ) +} + +fn build_supervisor_prelude_rules() -> BTreeMap> { + let mut rules: BTreeMap> = BTreeMap::new(); + + for syscall in [ + libc::SYS_mount, + libc::SYS_fsopen, + libc::SYS_fsconfig, + libc::SYS_fsmount, + libc::SYS_fspick, + libc::SYS_move_mount, + libc::SYS_open_tree, + libc::SYS_pivot_root, + libc::SYS_umount2, + libc::SYS_bpf, + libc::SYS_perf_event_open, + libc::SYS_userfaultfd, + libc::SYS_init_module, + libc::SYS_finit_module, + libc::SYS_delete_module, + libc::SYS_kexec_load, + libc::SYS_kexec_file_load, + ] { + rules.entry(syscall).or_default(); + } + + rules +} + +fn set_no_new_privs() -> Result<()> { let rc = unsafe { libc::prctl(libc::PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) }; if rc != 0 { return Err(miette::miette!( @@ -39,25 +98,19 @@ pub fn apply(policy: &SandboxPolicy) -> Result<()> { )); } - apply_runtime_filters(&main_filter, &clone3_filter)?; - Ok(()) } -fn build_filter(allow_inet: bool) -> Result { - let rules = build_filter_rules(allow_inet)?; - +fn compile_filter( + rules: BTreeMap>, + blocked_action: SeccompAction, +) -> Result { let arch = std::env::consts::ARCH .try_into() .map_err(|_| miette::miette!("Unsupported architecture for seccomp"))?; - let filter = SeccompFilter::new( - rules, - SeccompAction::Allow, - SeccompAction::Errno(libc::EPERM as u32), - arch, - ) - .into_diagnostic()?; + let filter = + SeccompFilter::new(rules, SeccompAction::Allow, blocked_action, arch).into_diagnostic()?; filter.try_into().into_diagnostic() } @@ -76,20 +129,7 @@ fn build_filter(allow_inet: bool) -> Result { fn build_clone3_filter() -> Result { let mut rules: BTreeMap> = BTreeMap::new(); rules.entry(libc::SYS_clone3).or_default(); - - let arch = std::env::consts::ARCH - .try_into() - .map_err(|_| miette::miette!("Unsupported architecture for seccomp"))?; - - let filter = SeccompFilter::new( - rules, - SeccompAction::Allow, - SeccompAction::Errno(libc::ENOSYS as u32), - arch, - ) - .into_diagnostic()?; - - filter.try_into().into_diagnostic() + compile_filter(rules, SeccompAction::Errno(libc::ENOSYS as u32)) } /// Install the sandbox seccomp filters in the required order. @@ -261,6 +301,15 @@ mod tests { assert!(filter.is_ok(), "build_filter(false) should succeed"); } + #[test] + fn build_supervisor_prelude_filter_compiles() { + let filter = build_supervisor_prelude_filter(); + assert!( + filter.is_ok(), + "build_supervisor_prelude_filter() should succeed" + ); + } + #[test] fn add_masked_arg_rule_creates_entry() { let mut rules: BTreeMap> = BTreeMap::new(); @@ -342,6 +391,57 @@ mod tests { } } + #[test] + fn supervisor_prelude_blocks_expected_syscalls() { + let filter_rules = build_supervisor_prelude_rules(); + + for syscall in [ + libc::SYS_mount, + libc::SYS_fsopen, + libc::SYS_fsconfig, + libc::SYS_fsmount, + libc::SYS_fspick, + libc::SYS_move_mount, + libc::SYS_open_tree, + libc::SYS_pivot_root, + libc::SYS_umount2, + libc::SYS_bpf, + libc::SYS_perf_event_open, + libc::SYS_userfaultfd, + libc::SYS_init_module, + libc::SYS_finit_module, + libc::SYS_delete_module, + libc::SYS_kexec_load, + libc::SYS_kexec_file_load, + ] { + assert!( + filter_rules.contains_key(&syscall), + "syscall {syscall} should be in the supervisor prelude rules" + ); + assert!( + filter_rules[&syscall].is_empty(), + "syscall {syscall} should be unconditionally blocked in the supervisor prelude" + ); + } + } + + #[test] + fn supervisor_prelude_keeps_required_setup_syscalls_available() { + let filter_rules = build_supervisor_prelude_rules(); + + for syscall in [ + libc::SYS_setns, + libc::SYS_clone, + libc::SYS_unshare, + libc::SYS_ptrace, + ] { + assert!( + !filter_rules.contains_key(&syscall), + "syscall {syscall} should remain available during supervisor startup" + ); + } + } + #[test] fn clone3_filter_compiles_and_blocks_clone3() { let bpf = build_clone3_filter(); @@ -448,6 +548,46 @@ mod tests { unsafe { assert_blocked_in_child(&filter, libc::SYS_setns, libc::EPERM) }; } + #[test] + fn behavioral_supervisor_prelude_mount_blocked() { + let pid = unsafe { libc::fork() }; + assert!(pid >= 0, "fork failed"); + if pid == 0 { + unsafe { + if let Err(err) = apply_supervisor_prelude() { + let msg = format!("failed to install supervisor prelude: {err}\n"); + libc::write(2, msg.as_ptr().cast(), msg.len()); + libc::_exit(1); + } + let ret = libc::syscall( + libc::SYS_mount, + std::ptr::null::(), + std::ptr::null::(), + std::ptr::null::(), + 0 as libc::c_ulong, + std::ptr::null::(), + ); + let errno = *libc::__errno_location(); + if ret == -1 && errno == libc::EPERM { + libc::_exit(0); + } else { + let msg = format!( + "mount: expected EPERM after supervisor prelude, got ret={ret} errno={errno}\n" + ); + libc::write(2, msg.as_ptr().cast(), msg.len()); + libc::_exit(1); + } + } + } + + let mut status: libc::c_int = 0; + unsafe { libc::waitpid(pid, &mut status, 0) }; + assert!( + unsafe { libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0 }, + "mount should be blocked by the supervisor prelude filter" + ); + } + #[test] fn behavioral_clone3_returns_enosys() { // clone3 uses a separate filter that returns ENOSYS (not EPERM) so diff --git a/crates/openshell-sandbox/src/sandbox/mod.rs b/crates/openshell-sandbox/src/sandbox/mod.rs index f7b0373383..a153b7b9e5 100644 --- a/crates/openshell-sandbox/src/sandbox/mod.rs +++ b/crates/openshell-sandbox/src/sandbox/mod.rs @@ -38,3 +38,17 @@ pub fn apply(policy: &SandboxPolicy, workdir: Option<&str>) -> Result<()> { Ok(()) } } + +/// Apply seccomp hardening for the long-lived supervisor process itself. +#[cfg_attr(not(target_os = "linux"), allow(clippy::unnecessary_wraps))] +pub fn apply_supervisor_startup_hardening() -> Result<()> { + #[cfg(target_os = "linux")] + { + linux::apply_supervisor_prelude() + } + + #[cfg(not(target_os = "linux"))] + { + Ok(()) + } +} diff --git a/docs/security/best-practices.mdx b/docs/security/best-practices.mdx index c28d6a68ff..45a521fbf8 100644 --- a/docs/security/best-practices.mdx +++ b/docs/security/best-practices.mdx @@ -194,12 +194,13 @@ The sandbox process runs as a non-root user after explicit privilege dropping. ### Seccomp Filters -A BPF seccomp filter restricts which socket domains the sandbox process can use and blocks a denylist of syscalls that enable container escape, privilege escalation, or host observation. The sandbox sets `PR_SET_NO_NEW_PRIVS` before applying the filter. +OpenShell applies seccomp in two phases. A narrow supervisor-startup prelude runs before CLI parsing and async runtime initialization, then the child process receives the broader runtime seccomp filter after privilege drop. | Aspect | Detail | |---|---| +| Startup prelude | After privileged bootstrap helpers complete, the supervisor sets `PR_SET_NO_NEW_PRIVS` and synchronizes a seccomp filter across all runtime threads that blocks `mount`, the new mount API syscalls, `pivot_root`, `umount2`, `bpf`, `perf_event_open`, `userfaultfd`, module-loading syscalls, and kexec. This closes the long-lived privileged remount and kernel-surface window while leaving required setup syscalls such as `setns` available. | | Socket domains | The filter allows `AF_INET` and `AF_INET6` (for proxy communication) and blocks `AF_NETLINK`, `AF_PACKET`, `AF_BLUETOOTH`, and `AF_VSOCK` with `EPERM`. | -| Unconditional syscall blocks | `memfd_create`, `ptrace`, `bpf`, `process_vm_readv`, `process_vm_writev`, `pidfd_open`, `pidfd_getfd`, `pidfd_send_signal`, `io_uring_setup`, `mount`, `fsopen`, `fsconfig`, `fsmount`, `fspick`, `move_mount`, `open_tree`, `setns`, `umount2`, `pivot_root`, `userfaultfd`, `perf_event_open`. | +| Runtime unconditional syscall blocks | `memfd_create`, `ptrace`, `bpf`, `process_vm_readv`, `process_vm_writev`, `pidfd_open`, `pidfd_getfd`, `pidfd_send_signal`, `io_uring_setup`, `mount`, `fsopen`, `fsconfig`, `fsmount`, `fspick`, `move_mount`, `open_tree`, `setns`, `umount2`, `pivot_root`, `userfaultfd`, `perf_event_open`. | | Conditional syscall blocks | `execveat` with `AT_EMPTY_PATH`, `unshare` and `clone` with `CLONE_NEWUSER`, and `seccomp(SECCOMP_SET_MODE_FILTER)` are denied with `EPERM`. | | What you can change | This is not a user-facing knob. OpenShell enforces it automatically. | | Risk if relaxed | The blocked syscalls support container escape (`mount`, `pivot_root`, `move_mount`, namespace creation), cross-process observation (`ptrace`, `process_vm_readv`, `pidfd_*`), raw kernel bypass (`bpf`, `io_uring_setup`, `perf_event_open`), and filter evasion (`seccomp`, `userfaultfd`). | @@ -208,13 +209,15 @@ A BPF seccomp filter restricts which socket domains the sandbox process can use ### Enforcement Application Order The sandbox supervisor applies enforcement in a specific order during process startup. -This ordering is intentional: privilege dropping needs `/etc/group` and `/etc/passwd`, which Landlock subsequently restricts. - -1. Network namespace entry (`setns`). -2. Privilege drop (`initgroups` + `setgid` + `setuid`). -3. Core-dump hardening (`RLIMIT_CORE=0`, plus `PR_SET_DUMPABLE=0` on Linux). -4. Landlock filesystem restrictions. -5. Seccomp socket domain filters. +This ordering is intentional: named network-namespace setup still relies on privileged helpers, and privilege dropping still needs `/etc/group` and `/etc/passwd`, which Landlock subsequently restricts. + +1. Privileged supervisor bootstrap helpers, including network-namespace setup and optional `iptables` probes. +2. Supervisor startup prelude seccomp (`PR_SET_NO_NEW_PRIVS` plus the early syscall denylist) synchronized across runtime threads. +3. Network namespace entry (`setns`) in child `pre_exec`. +4. Privilege drop (`initgroups` + `setgid` + `setuid`). +5. Core-dump hardening (`RLIMIT_CORE=0`, plus `PR_SET_DUMPABLE=0` on Linux). +6. Landlock filesystem restrictions. +7. Runtime seccomp socket domain and syscall filters. ## Inference Controls From b39f5aaa0c1b530e122ebedbab4eb02566b02e3c Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Mon, 20 Apr 2026 15:15:01 -0700 Subject: [PATCH 016/142] feat(install-vm): install gateway + vm driver, add --driver-dir resolution (#887) --- architecture/gateway.md | 4 +- crates/openshell-driver-vm/README.md | 4 +- crates/openshell-driver-vm/start.sh | 1 - crates/openshell-server/src/cli.rs | 12 +- crates/openshell-server/src/compute/vm.rs | 142 ++++++++++-- install-vm.sh | 264 ++++++++++++++++------ 6 files changed, 322 insertions(+), 105 deletions(-) diff --git a/architecture/gateway.md b/architecture/gateway.md index 02f487050d..294506e57d 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -99,7 +99,7 @@ The gateway boots in `main()` (`crates/openshell-server/src/main.rs`) and procee 1. Connect to the persistence store (`Store::connect`), which auto-detects SQLite vs Postgres from the URL prefix and runs migrations. 2. Create `ComputeRuntime` with a `ComputeDriver` implementation selected by `OPENSHELL_DRIVERS`: - `kubernetes` wraps `KubernetesComputeDriver` in `ComputeDriverService`, so the gateway uses the `openshell.compute.v1.ComputeDriver` RPC surface even without transport. - - `vm` spawns the sibling `openshell-driver-vm` binary as a local compute-driver process, connects to it over a Unix domain socket, and keeps the libkrun/rootfs runtime out of the gateway binary. + - `vm` spawns the standalone `openshell-driver-vm` binary as a local compute-driver process, resolves it from `--driver-dir`, conventional libexec install paths, or a sibling of the gateway binary, connects to it over a Unix domain socket, and keeps the libkrun/rootfs runtime out of the gateway binary. 3. Build `ServerState` (shared via `Arc` across all handlers). 4. **Spawn background tasks**: - `ComputeRuntime::spawn_watchers` -- consumes the compute-driver watch stream, republishes platform events, and runs a periodic `ListSandboxes` snapshot reconcile so the store-backed public sandbox reads stay aligned with the compute driver. @@ -128,7 +128,7 @@ All configuration is via CLI flags with environment variable fallbacks. The `--d | `--grpc-endpoint` | `OPENSHELL_GRPC_ENDPOINT` | None | gRPC endpoint reachable from within the cluster (for sandbox callbacks) | | `--drivers` | `OPENSHELL_DRIVERS` | `kubernetes` | Compute backend to use. Current options are `kubernetes` and `vm`. | | `--vm-driver-state-dir` | `OPENSHELL_VM_DRIVER_STATE_DIR` | `target/openshell-vm-driver` | Host directory for VM sandbox rootfs, console logs, and runtime state | -| `--vm-compute-driver-bin` | `OPENSHELL_VM_COMPUTE_DRIVER_BIN` | sibling `openshell-driver-vm` binary | Local VM compute-driver process spawned by the gateway | +| `--driver-dir` | `OPENSHELL_DRIVER_DIR` | unset | Override directory for `openshell-driver-vm`. When unset, the gateway searches `~/.local/libexec/openshell`, `/usr/local/libexec/openshell`, `/usr/local/libexec`, then a sibling binary. | | `--vm-krun-log-level` | `OPENSHELL_VM_KRUN_LOG_LEVEL` | `1` | libkrun log level for VM helper processes | | `--vm-driver-vcpus` | `OPENSHELL_VM_DRIVER_VCPUS` | `2` | Default vCPU count for VM sandboxes | | `--vm-driver-mem-mib` | `OPENSHELL_VM_DRIVER_MEM_MIB` | `2048` | Default memory allocation for VM sandboxes in MiB | diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index 6b874eb105..a95462695c 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -90,7 +90,7 @@ target/debug/openshell-gateway \ --vm-driver-state-dir $PWD/target/openshell-vm-driver-dev ``` -The gateway discovers `openshell-driver-vm` as a sibling of its own binary. Pass `--vm-compute-driver-bin /path/to/openshell-driver-vm` (or set `OPENSHELL_VM_COMPUTE_DRIVER_BIN`) to override. +The gateway resolves `openshell-driver-vm` in this order: `--driver-dir`, conventional install locations (`~/.local/libexec/openshell`, `/usr/local/libexec/openshell`, `/usr/local/libexec`), then a sibling of the gateway binary. ## Flags @@ -99,7 +99,7 @@ The gateway discovers `openshell-driver-vm` as a sibling of its own binary. Pass | `--drivers vm` | `OPENSHELL_DRIVERS` | `kubernetes` | Select the VM compute driver. | | `--grpc-endpoint URL` | `OPENSHELL_GRPC_ENDPOINT` | — | Required. URL the sandbox guest calls back to. Use a host alias that resolves to the gateway's host from inside the VM (gvproxy answers `host.containers.internal` and `host.openshell.internal` to `192.168.127.1`). | | `--vm-driver-state-dir DIR` | `OPENSHELL_VM_DRIVER_STATE_DIR` | `target/openshell-vm-driver` | Per-sandbox rootfs, console logs, and the `compute-driver.sock` UDS. | -| `--vm-compute-driver-bin PATH` | `OPENSHELL_VM_COMPUTE_DRIVER_BIN` | sibling of gateway binary | Override the driver binary path. | +| `--driver-dir DIR` | `OPENSHELL_DRIVER_DIR` | unset | Override the directory searched for `openshell-driver-vm`. | | `--vm-driver-vcpus N` | `OPENSHELL_VM_DRIVER_VCPUS` | `2` | vCPUs per sandbox. | | `--vm-driver-mem-mib N` | `OPENSHELL_VM_DRIVER_MEM_MIB` | `2048` | Memory per sandbox, in MiB. | | `--vm-krun-log-level N` | `OPENSHELL_VM_KRUN_LOG_LEVEL` | `1` | libkrun verbosity (0–5). | diff --git a/crates/openshell-driver-vm/start.sh b/crates/openshell-driver-vm/start.sh index aa98874607..155136c785 100755 --- a/crates/openshell-driver-vm/start.sh +++ b/crates/openshell-driver-vm/start.sh @@ -57,7 +57,6 @@ export OPENSHELL_SSH_GATEWAY_HOST="${OPENSHELL_SSH_GATEWAY_HOST:-127.0.0.1}" export OPENSHELL_SSH_GATEWAY_PORT="${OPENSHELL_SSH_GATEWAY_PORT:-${SERVER_PORT}}" export OPENSHELL_SSH_HANDSHAKE_SECRET="${OPENSHELL_SSH_HANDSHAKE_SECRET:-dev-vm-driver-secret}" export OPENSHELL_VM_DRIVER_STATE_DIR="${STATE_DIR}" -export OPENSHELL_VM_COMPUTE_DRIVER_BIN="${OPENSHELL_VM_COMPUTE_DRIVER_BIN:-${ROOT}/target/debug/openshell-driver-vm}" echo "==> Starting OpenShell server with VM compute driver" exec "${ROOT}/target/debug/openshell-gateway" diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index ba94250368..a36ad05e7e 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -121,9 +121,13 @@ struct Args { )] vm_driver_state_dir: PathBuf, - /// VM compute-driver binary spawned by the gateway. - #[arg(long, env = "OPENSHELL_VM_COMPUTE_DRIVER_BIN")] - vm_compute_driver_bin: Option, + /// Directory searched for compute-driver binaries (e.g. + /// `openshell-driver-vm`) when an explicit binary override isn't + /// configured. When unset, the gateway searches + /// `$HOME/.local/libexec/openshell`, `/usr/local/libexec/openshell`, + /// `/usr/local/libexec`, then a sibling of the gateway binary. + #[arg(long, env = "OPENSHELL_DRIVER_DIR")] + driver_dir: Option, /// libkrun log level used by the VM helper. #[arg( @@ -262,7 +266,7 @@ async fn run_from_args(args: Args) -> Result<()> { let vm_config = VmComputeConfig { state_dir: args.vm_driver_state_dir, - compute_driver_bin: args.vm_compute_driver_bin, + driver_dir: args.driver_dir, krun_log_level: args.vm_krun_log_level, vcpus: args.vm_vcpus, mem_mib: args.vm_mem_mib, diff --git a/crates/openshell-server/src/compute/vm.rs b/crates/openshell-server/src/compute/vm.rs index d0f397b01e..622bc393e5 100644 --- a/crates/openshell-server/src/compute/vm.rs +++ b/crates/openshell-server/src/compute/vm.rs @@ -51,15 +51,17 @@ use tonic::transport::Endpoint; #[cfg(unix)] use tower::service_fn; +const DRIVER_BIN_NAME: &str = "openshell-driver-vm"; + /// Configuration for launching and talking to the VM compute driver. #[derive(Debug, Clone)] pub struct VmComputeConfig { /// Working directory for VM driver sandbox state. pub state_dir: PathBuf, - /// Optional override for the `openshell-driver-vm` binary path. - /// When `None`, the gateway resolves a sibling of its own executable. - pub compute_driver_bin: Option, + /// Directory to search for compute-driver binaries before the gateway + /// falls back to its conventional install paths and sibling binary. + pub driver_dir: Option, /// libkrun log level used by the VM driver helper. pub krun_log_level: u32, @@ -104,13 +106,24 @@ impl VmComputeConfig { pub const fn default_mem_mib() -> u32 { 2048 } + + #[must_use] + fn default_driver_search_dirs(home: Option) -> Vec { + let mut dirs = Vec::new(); + if let Some(home) = home { + dirs.push(home.join(".local").join("libexec").join("openshell")); + } + push_unique_path(&mut dirs, PathBuf::from("/usr/local/libexec/openshell")); + push_unique_path(&mut dirs, PathBuf::from("/usr/local/libexec")); + dirs + } } impl Default for VmComputeConfig { fn default() -> Self { Self { state_dir: Self::default_state_dir(), - compute_driver_bin: None, + driver_dir: None, krun_log_level: Self::default_krun_log_level(), vcpus: Self::default_vcpus(), mem_mib: Self::default_mem_mib(), @@ -129,31 +142,66 @@ pub(crate) struct VmGuestTlsPaths { pub(crate) key: PathBuf, } -/// Resolve the `openshell-driver-vm` binary path, falling back to a sibling -/// of the gateway's own executable when an override is not supplied. +/// Resolve the `openshell-driver-vm` binary path. +/// +/// Resolution order: +/// 1. `{driver_dir}/openshell-driver-vm`, where `driver_dir` comes from +/// `--driver-dir` / `OPENSHELL_DRIVER_DIR`. +/// 2. Conventional install directories: +/// `~/.local/libexec/openshell`, `/usr/local/libexec/openshell`, +/// `/usr/local/libexec`. +/// 3. Sibling of the gateway's own executable (last-resort fallback so +/// local development builds still work out of the box). pub(crate) fn resolve_compute_driver_bin(vm_config: &VmComputeConfig) -> Result { - let path = if let Some(path) = vm_config.compute_driver_bin.clone() { - path - } else { - let current_exe = std::env::current_exe() - .map_err(|e| Error::config(format!("failed to resolve current executable: {e}")))?; - let Some(parent) = current_exe.parent() else { - return Err(Error::config(format!( - "current executable '{}' has no parent directory", - current_exe.display() - ))); - }; - parent.join("openshell-driver-vm") - }; + let mut searched: Vec = Vec::new(); + + // 1. Configured driver directory, or the conventional install locations + // when no explicit override is configured. + for dir in resolve_driver_search_dirs(vm_config) { + let candidate = dir.join(DRIVER_BIN_NAME); + if candidate.is_file() { + return Ok(candidate); + } + push_unique_path(&mut searched, candidate); + } - if !path.is_file() { + // 2. Sibling-of-gateway fallback. + let current_exe = std::env::current_exe() + .map_err(|e| Error::config(format!("failed to resolve current executable: {e}")))?; + let Some(parent) = current_exe.parent() else { return Err(Error::config(format!( - "vm compute driver binary '{}' does not exist; set --vm-compute-driver-bin or OPENSHELL_VM_COMPUTE_DRIVER_BIN", - path.display() + "current executable '{}' has no parent directory", + current_exe.display() ))); + }; + let sibling = parent.join(DRIVER_BIN_NAME); + if sibling.is_file() { + return Ok(sibling); + } + push_unique_path(&mut searched, sibling); + + let searched_display = searched + .iter() + .map(|p| format!("'{}'", p.display())) + .collect::>() + .join(", "); + Err(Error::config(format!( + "vm compute driver binary not found (searched {searched_display}); install it under --driver-dir / OPENSHELL_DRIVER_DIR, a conventional libexec path such as ~/.local/libexec/openshell or /usr/local/libexec{{,/openshell}}, or place it next to the gateway binary" + ))) +} + +fn resolve_driver_search_dirs(vm_config: &VmComputeConfig) -> Vec { + if let Some(dir) = vm_config.driver_dir.clone() { + vec![dir] + } else { + VmComputeConfig::default_driver_search_dirs(std::env::var_os("HOME").map(PathBuf::from)) } +} - Ok(path) +fn push_unique_path(paths: &mut Vec, path: PathBuf) { + if !paths.iter().any(|existing| existing == &path) { + paths.push(path); + } } /// Path of the Unix domain socket the driver will listen on. @@ -353,10 +401,56 @@ async fn connect_compute_driver(socket_path: &std::path::Path) -> Result&1 | sed -n 's/^.*Location: \([^ ]*\).*/\1/p' | tail -1 fi } @@ -100,6 +119,7 @@ validate_download_origin() { # Platform detection # --------------------------------------------------------------------------- +# Both binaries ship the same set of triples under the same naming scheme. get_target() { _arch="$(uname -m)" _os="$(uname -s)" @@ -149,17 +169,27 @@ verify_checksum() { } # --------------------------------------------------------------------------- -# Install location +# Install locations # --------------------------------------------------------------------------- -get_install_dir() { - if [ -n "${OPENSHELL_VM_INSTALL_DIR:-}" ]; then - echo "$OPENSHELL_VM_INSTALL_DIR" +get_gateway_install_dir() { + if [ -n "${OPENSHELL_INSTALL_DIR:-}" ]; then + echo "$OPENSHELL_INSTALL_DIR" else echo "${HOME}/.local/bin" fi } +# Default per-user install dir for the VM compute driver. Newer gateways also +# auto-discover conventional system installs under `/usr/local/libexec`. +get_driver_install_dir() { + if [ -n "${OPENSHELL_DRIVER_DIR:-}" ]; then + echo "$OPENSHELL_DRIVER_DIR" + else + echo "${HOME}/.local/libexec/openshell" + fi +} + is_on_path() { case ":${PATH}:" in *":$1:"*) return 0 ;; @@ -168,10 +198,11 @@ is_on_path() { } # --------------------------------------------------------------------------- -# macOS codesign +# macOS codesign — the VM driver runs libkrun and needs the hypervisor +# entitlement. The gateway does not. # --------------------------------------------------------------------------- -codesign_binary() { +codesign_driver_vm() { _binary="$1" _cs_tmpdir="$2" # reuse caller's tmpdir for cleanup-safe temp files @@ -180,11 +211,11 @@ codesign_binary() { fi if ! has_cmd codesign; then - warn "codesign not found; the binary will fail without the Hypervisor entitlement" + warn "codesign not found; ${DRIVER_VM_BIN} will fail without the Hypervisor entitlement" return 0 fi - info "codesigning with Hypervisor entitlement..." + info "codesigning ${DRIVER_VM_BIN} with Hypervisor entitlement..." _entitlements="${_cs_tmpdir}/entitlements.plist" cat > "$_entitlements" <<'PLIST' @@ -199,16 +230,66 @@ PLIST codesign --entitlements "$_entitlements" --force -s - "$_binary" } +# --------------------------------------------------------------------------- +# Download + install a single binary release asset +# --------------------------------------------------------------------------- + +# Args: +# $1 binary name (e.g. openshell-gateway) +# $2 release tag (e.g. dev, vm-dev) +# $3 target triple (e.g. aarch64-apple-darwin) +# $4 checksums filename in the release (e.g. openshell-gateway-checksums-sha256.txt) +# $5 destination directory +# $6 tmp working dir (caller-owned; will be cleaned up outside) +install_release_binary() { + _bin="$1" + _tag="$2" + _target="$3" + _checksums_name="$4" + _dest_dir="$5" + _work_dir="$6" + + _filename="${_bin}-${_target}.tar.gz" + _download_url="${GITHUB_URL}/releases/download/${_tag}/${_filename}" + _checksums_url="${GITHUB_URL}/releases/download/${_tag}/${_checksums_name}" + + info "downloading ${_bin} from release '${_tag}' (${_target})..." + + validate_download_origin "$_download_url" + + if ! download "$_download_url" "${_work_dir}/${_filename}"; then + error "failed to download ${_download_url}" + fi + + if ! download "$_checksums_url" "${_work_dir}/${_bin}-checksums.txt"; then + error "failed to download checksums file from ${_checksums_url}" + fi + + info "verifying ${_bin} checksum..." + if ! verify_checksum "${_work_dir}/${_filename}" "${_work_dir}/${_bin}-checksums.txt" "$_filename"; then + error "checksum verification failed for ${_filename}" + fi + + info "extracting ${_bin}..." + tar -xzf "${_work_dir}/${_filename}" -C "${_work_dir}" --no-same-owner --no-same-permissions "${_bin}" + + # Install into destination dir, escalating with sudo if needed. + if mkdir -p "$_dest_dir" 2>/dev/null && [ -w "$_dest_dir" ]; then + install -m 755 "${_work_dir}/${_bin}" "${_dest_dir}/${_bin}" + else + info "elevated permissions required to install to ${_dest_dir}" + sudo mkdir -p "$_dest_dir" + sudo install -m 755 "${_work_dir}/${_bin}" "${_dest_dir}/${_bin}" + fi +} + # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- -main() { - for arg in "$@"; do - case "$arg" in - --help) - cat </dev/null || true - - if [ -w "$_install_dir" ] || mkdir -p "$_install_dir" 2>/dev/null; then - install -m 755 "${_tmpdir}/${APP_NAME}" "${_install_dir}/${APP_NAME}" - else - info "elevated permissions required to install to ${_install_dir}" - sudo mkdir -p "$_install_dir" - sudo install -m 755 "${_tmpdir}/${APP_NAME}" "${_install_dir}/${APP_NAME}" - fi - - codesign_binary "${_install_dir}/${APP_NAME}" "$_tmpdir" - - _installed_version="$("${_install_dir}/${APP_NAME}" --version 2>/dev/null || echo "${RELEASE_TAG}")" - info "installed ${_installed_version} to ${_install_dir}/${APP_NAME}" - - # If the install directory isn't on PATH, print instructions - if ! is_on_path "$_install_dir"; then + # 1. Gateway — from the rolling `dev` release. + install_release_binary \ + "$GATEWAY_BIN" \ + "$GATEWAY_RELEASE_TAG" \ + "$_target" \ + "${GATEWAY_BIN}-checksums-sha256.txt" \ + "$_gateway_dir" \ + "$_tmpdir" + + # 2. VM compute driver — from the rolling `vm-dev` release. Shares the + # checksum file with openshell-vm (`vm-binary-checksums-sha256.txt`). + install_release_binary \ + "$DRIVER_VM_BIN" \ + "$DRIVER_VM_RELEASE_TAG" \ + "$_target" \ + "vm-binary-checksums-sha256.txt" \ + "$_driver_dir" \ + "$_tmpdir" + + codesign_driver_vm "${_driver_dir}/${DRIVER_VM_BIN}" "$_tmpdir" + + _gateway_version="$("${_gateway_dir}/${GATEWAY_BIN}" --version 2>/dev/null || echo "${GATEWAY_RELEASE_TAG}")" + info "installed ${_gateway_version} to ${_gateway_dir}/${GATEWAY_BIN}" + info "installed ${DRIVER_VM_BIN} to ${_driver_dir}/${DRIVER_VM_BIN}" + + # Warn if the gateway dir isn't on PATH. + if ! is_on_path "$_gateway_dir"; then echo "" - info "${_install_dir} is not on your PATH." + info "${_gateway_dir} is not on your PATH." info "" info "Add it by appending the following to your shell configuration file" info "(e.g. ~/.bashrc, ~/.zshrc, or ~/.config/fish/config.fish):" @@ -290,17 +373,54 @@ EOF _current_shell="$(basename "${SHELL:-sh}" 2>/dev/null || echo "sh")" case "$_current_shell" in - fish) - info " fish_add_path ${_install_dir}" - ;; - *) - info " export PATH=\"${_install_dir}:\$PATH\"" - ;; + fish) info " fish_add_path ${_gateway_dir}" ;; + *) info " export PATH=\"${_gateway_dir}:\$PATH\"" ;; esac info "" info "Then restart your shell or run the command above in your current session." fi + + # --------------------------------------------------------------------------- + # Next steps — print a working command to start the gateway. + # + # The VM compute driver requires: + # * --driver-dir — only needed when the driver is installed + # outside the built-in search paths: + # ~/.local/libexec/openshell, + # /usr/local/libexec/openshell, + # /usr/local/libexec, or next to the gateway. + # * --grpc-endpoint — URL the VM guest uses to call the gateway + # back. Loopback is accepted; scheme must + # match TLS mode. + # * --ssh-handshake-secret — shared secret for gateway↔sandbox SSH. + # --------------------------------------------------------------------------- + + echo "" + info "Next steps — start the gateway with the VM compute driver:" + echo "" + + _driver_dir_arg="" + case "$_driver_dir" in + "${HOME}/.local/libexec/openshell"|"/usr/local/libexec/openshell"|"/usr/local/libexec") ;; + *) + _driver_dir_arg=" --driver-dir '${_driver_dir}' \\ +" + ;; + esac + + cat >&2 < Date: Mon, 20 Apr 2026 16:44:48 -0700 Subject: [PATCH 017/142] fix(cli): sandbox get returns currently active runtime policy (#880) * feat(cli): Allow printing only the policy when retrieving sandbox info * feat(cli): Indicate policy source and revision when retrieving sandboxes * cargo fmt --- .agents/skills/openshell-cli/cli-reference.md | 6 +- crates/openshell-cli/src/main.rs | 8 ++- crates/openshell-cli/src/run.rs | 62 +++++++++++++++++-- .../sandbox_name_fallback_integration.rs | 38 +++++++++--- docs/sandboxes/manage-sandboxes.mdx | 8 ++- 5 files changed, 107 insertions(+), 15 deletions(-) diff --git a/.agents/skills/openshell-cli/cli-reference.md b/.agents/skills/openshell-cli/cli-reference.md index c9c5450a00..bdbb35572c 100644 --- a/.agents/skills/openshell-cli/cli-reference.md +++ b/.agents/skills/openshell-cli/cli-reference.md @@ -181,7 +181,11 @@ Create a sandbox, wait for readiness, then connect or execute the trailing comma ### `openshell sandbox get ` -Show sandbox details (id, name, namespace, phase, policy). +Show sandbox details (id, name, namespace, phase) and the **active** policy from the gateway (same source whether policy is sandbox-scoped or global). Metadata includes **Policy source** (`sandbox` or `global`) and **Revision** (global policy row when source is global, otherwise sandbox policy row). + +| Flag | Description | +|------|-------------| +| `--policy-only` | Print only the active policy YAML to stdout (same policy as above; use for scripts and piping) | ### `openshell sandbox list` diff --git a/crates/openshell-cli/src/main.rs b/crates/openshell-cli/src/main.rs index 8474afd15a..6239978d75 100644 --- a/crates/openshell-cli/src/main.rs +++ b/crates/openshell-cli/src/main.rs @@ -1208,6 +1208,10 @@ enum SandboxCommands { /// Sandbox name (defaults to last-used sandbox). #[arg(add = ArgValueCompleter::new(completers::complete_sandbox_names))] name: Option, + + /// Print only the active policy YAML (same policy as the default view; stdout only). + #[arg(long)] + policy_only: bool, }, /// List sandboxes. @@ -2461,9 +2465,9 @@ async fn main() -> Result<()> { | SandboxCommands::Download { .. } => { unreachable!() } - SandboxCommands::Get { name } => { + SandboxCommands::Get { name, policy_only } => { let name = resolve_sandbox_name(name, &ctx.name)?; - run::sandbox_get(endpoint, &name, &tls).await?; + run::sandbox_get(endpoint, &name, policy_only, &tls).await?; } SandboxCommands::List { limit, diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index 60e28ac7e1..ba25488b8b 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -2734,7 +2734,16 @@ pub async fn sandbox_sync_command( } /// Fetch a sandbox by name. -pub async fn sandbox_get(server: &str, name: &str, tls: &TlsOptions) -> Result<()> { +/// +/// Policy always comes from [`GetSandboxConfig`] (effective active policy, sandbox +/// or global). With `policy_only`, prints only that YAML to stdout; otherwise +/// prints sandbox metadata and the same policy with formatted YAML. +pub async fn sandbox_get( + server: &str, + name: &str, + policy_only: bool, + tls: &TlsOptions, +) -> Result<()> { let mut client = grpc_client(server, tls).await?; let response = client @@ -2748,6 +2757,26 @@ pub async fn sandbox_get(server: &str, name: &str, tls: &TlsOptions) -> Result<( .sandbox .ok_or_else(|| miette::miette!("sandbox missing from response"))?; + let config = client + .get_sandbox_config(GetSandboxConfigRequest { + sandbox_id: sandbox.id.clone(), + }) + .await + .into_diagnostic()? + .into_inner(); + + if policy_only { + let Some(ref policy) = config.policy else { + return Err(miette::miette!( + "no active policy configured for this sandbox" + )); + }; + let yaml_str = openshell_policy::serialize_sandbox_policy(policy) + .wrap_err("failed to serialize policy to YAML")?; + print!("{yaml_str}"); + return Ok(()); + } + println!("{}", "Sandbox:".cyan().bold()); println!(); println!(" {} {}", "Id:".dimmed(), sandbox.id); @@ -2755,9 +2784,34 @@ pub async fn sandbox_get(server: &str, name: &str, tls: &TlsOptions) -> Result<( println!(" {} {}", "Namespace:".dimmed(), sandbox.namespace); println!(" {} {}", "Phase:".dimmed(), phase_name(sandbox.phase)); - if let Some(spec) = &sandbox.spec - && let Some(policy) = &spec.policy - { + let policy_from_global = config.policy_source == PolicySource::Global as i32; + println!( + " {} {}", + "Policy source:".dimmed(), + if policy_from_global { + "global" + } else { + "sandbox" + } + ); + let revision = if policy_from_global { + if config.global_policy_version > 0 { + Some(config.global_policy_version) + } else if config.version > 0 { + Some(config.version) + } else { + None + } + } else if config.version > 0 { + Some(config.version) + } else { + None + }; + if let Some(rev) = revision { + println!(" {} {}", "Revision:".dimmed(), rev); + } + + if let Some(ref policy) = config.policy { println!(); print_sandbox_policy(policy); } diff --git a/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs b/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs index fbadec4c34..885f659d00 100644 --- a/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs +++ b/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs @@ -12,8 +12,8 @@ use openshell_core::proto::{ GetProviderRequest, GetSandboxConfigRequest, GetSandboxConfigResponse, GetSandboxProviderEnvironmentRequest, GetSandboxProviderEnvironmentResponse, GetSandboxRequest, HealthRequest, HealthResponse, ListProvidersRequest, ListProvidersResponse, - ListSandboxesRequest, ListSandboxesResponse, ProviderResponse, Sandbox, SandboxResponse, - SandboxStreamEvent, ServiceStatus, UpdateProviderRequest, WatchSandboxRequest, + ListSandboxesRequest, ListSandboxesResponse, ProviderResponse, Sandbox, SandboxPolicy, + SandboxResponse, SandboxStreamEvent, ServiceStatus, UpdateProviderRequest, WatchSandboxRequest, }; use rcgen::{ BasicConstraints, Certificate, CertificateParams, ExtendedKeyUsagePurpose, IsCa, KeyPair, @@ -134,9 +134,20 @@ impl OpenShell for TestOpenShell { async fn get_sandbox_config( &self, - _request: tonic::Request, + request: tonic::Request, ) -> Result, Status> { - Ok(Response::new(GetSandboxConfigResponse::default())) + let req = request.into_inner(); + assert_eq!( + req.sandbox_id, "test-id", + "sandbox_get --policy-only should pass the id from GetSandbox" + ); + Ok(Response::new(GetSandboxConfigResponse { + policy: Some(SandboxPolicy { + version: 1, + ..Default::default() + }), + ..Default::default() + })) } async fn get_gateway_config( @@ -432,7 +443,7 @@ async fn run_server() -> TestServer { async fn sandbox_get_sends_correct_name() { let ts = run_server().await; - run::sandbox_get(&ts.endpoint, "my-sandbox", &ts.tls) + run::sandbox_get(&ts.endpoint, "my-sandbox", false, &ts.tls) .await .expect("sandbox_get should succeed"); @@ -444,6 +455,19 @@ async fn sandbox_get_sends_correct_name() { ); } +/// `sandbox_get` with `policy_only` calls `GetSandboxConfig` and prints YAML from the response. +#[tokio::test] +async fn sandbox_get_policy_only_round_trip() { + let ts = run_server().await; + + run::sandbox_get(&ts.endpoint, "my-sandbox", true, &ts.tls) + .await + .expect("sandbox_get with policy_only should succeed"); + + let recorded = ts.openshell.state.last_get_name.lock().await.clone(); + assert_eq!(recorded.as_deref(), Some("my-sandbox")); +} + /// End-to-end: save a last-used sandbox, load it back, then call `sandbox_get` /// with the resolved name. This validates the persistence + gRPC wiring. #[tokio::test] @@ -462,7 +486,7 @@ async fn sandbox_get_with_persisted_last_sandbox() { assert_eq!(resolved, "persisted-sb"); // Call sandbox_get with the resolved name. - run::sandbox_get(&ts.endpoint, &resolved, &ts.tls) + run::sandbox_get(&ts.endpoint, &resolved, false, &ts.tls) .await .expect("sandbox_get should succeed"); @@ -484,7 +508,7 @@ async fn explicit_name_takes_precedence_over_persisted() { // Persist one name, but supply a different one explicitly. save_last_sandbox("my-cluster", "old-sandbox").expect("save should succeed"); - run::sandbox_get(&ts.endpoint, "explicit-sandbox", &ts.tls) + run::sandbox_get(&ts.endpoint, "explicit-sandbox", false, &ts.tls) .await .expect("sandbox_get should succeed"); diff --git a/docs/sandboxes/manage-sandboxes.mdx b/docs/sandboxes/manage-sandboxes.mdx index 1be05e81e3..6d10efbd09 100644 --- a/docs/sandboxes/manage-sandboxes.mdx +++ b/docs/sandboxes/manage-sandboxes.mdx @@ -110,12 +110,18 @@ List all sandboxes: openshell sandbox list ``` -Get detailed information about a specific sandbox: +Get detailed information about a specific sandbox. The output lists **Policy source** (`sandbox` or `global`), **Revision** (the active policy’s row version for that source), and the formatted active policy YAML: ```shell openshell sandbox get my-sandbox ``` +Print only that policy YAML for scripting (same effective policy, no metadata): + +```shell +openshell sandbox get my-sandbox --policy-only +``` + Stream sandbox logs to monitor agent activity and diagnose policy decisions: ```shell From c960d480fd1dd7b037175aa911d04067494c00c2 Mon Sep 17 00:00:00 2001 From: "John T. Myers" <9696606+johntmyers@users.noreply.github.com> Date: Tue, 21 Apr 2026 07:08:07 -0700 Subject: [PATCH 018/142] fix(sandbox): canonicalize HTTP request-targets before L7 policy evaluation (#878) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(sandbox): canonicalize HTTP request-targets before L7 policy evaluation The L7 REST proxy evaluated OPA path rules against the raw request-target and forwarded the raw header block byte-for-byte to the upstream. The upstream normalized `..`, `%2e%2e`, and `//` before dispatching, so a compromised agent inside the sandbox could bypass any path-based allow rule by prefixing the blocked path with an allowed one (e.g. `GET /public/../secret` matches `/public/**` at the proxy but the upstream serves `/secret`). The inference-routing detection had the same normalization gap via `normalize_inference_path`, which only stripped scheme+authority and preserved dot-segments verbatim. - Add `crates/openshell-sandbox/src/l7/path.rs` with `canonicalize_request_target`. Percent-decodes (with uppercase hex re-emission), resolves `.` / `..` segments per RFC 3986 5.2.4, collapses doubled slashes, strips trailing `;params`, rejects fragments / raw non-ASCII / control bytes / encoded slashes (unless explicitly enabled per-endpoint), and returns a `rewritten` flag so callers know when to rewrite the outbound request line. - Wire the canonicalizer into `rest.rs::parse_http_request`. The canonical path is the `target` on `L7Request` that OPA sees, and when the canonical form differs from the raw input the request line is rebuilt in `raw_header` so the upstream dispatches on the exact bytes the policy evaluated. - Canonicalize the forward (plain HTTP proxy) path at `proxy.rs`'s second L7 evaluation site. A non-canonical request-target is rejected with 400 Bad Request and an OCSF `NetworkActivity` Fail event rather than silently passed through. - Replace the old `normalize_inference_path` body with a call to the canonicalizer. On canonicalization error the raw path is returned (so inference detection simply misses and the normal forward path handles and rejects). - Document the invariant in `sandbox-policy.rego`: `input.request.path` is pre-canonicalized so rules must not attempt defensive matching against `..` or `%2e%2e`. - Architecture doc (`architecture/sandbox.md`) lists the new module. - 24 new unit tests cover dot segments, percent-encoded dot segments, double-slash collapse, encoded-slash reject/opt-in, null/control byte rejection, legitimate percent-encoded bytes round-tripping, mixed-case percent normalization, fragment rejection, absolute-form stripping, empty / length-bounded / non-ASCII handling, and regression payloads for the specific bypasses above. Tracks OS-99. * fix(sandbox): close forward-proxy parser-differential and wire allow_encoded_slash Addresses two review findings on the L7 path canonicalization change. 1. `handle_forward_proxy` previously evaluated OPA on the canonical path but wrote the raw path to the upstream via `rewrite_forward_request`, leaving the parser-differential open for plain-HTTP forward-proxy traffic even though it was closed for the L7 tunnel path. `path` is now reassigned to the canonical form inside the L7 block before the outbound write, so the bytes dispatched to the upstream match what OPA evaluated. Covered by a new regression test against `rewrite_forward_request`. 2. `allow_encoded_slash` is now wired through the YAML policy schema, the proto `NetworkEndpoint` message, the Rego endpoint config, and the `L7EndpointConfig` → `RestProvider` canonicalization options. Operators can opt in per-endpoint for upstreams like GitLab that embed `%2F` in namespaced resource paths; the default remains strict. The `RestProvider` gains a constructor `RestProvider::with_options` so `relay_rest` constructs a provider with per-endpoint options while `relay_passthrough_with_credentials` retains strict defaults. Integration tests: - `parse_http_request_canonicalizes_target_and_rewrites_raw_header` drives a non-canonical request through the parser and asserts both `L7Request.target` (what OPA evaluates) and `raw_header` (what the upstream receives) are canonical. - `parse_http_request_canonicalization_preserves_query_string`, `parse_http_request_leaves_canonical_input_byte_for_byte`, `parse_http_request_rejects_traversal_above_root`, `parse_http_request_preserves_http_10_version_on_rewrite`. - `parse_http_request_accepts_encoded_slash_when_endpoint_opts_in` / `_rejects_encoded_slash_by_default` verify the `allow_encoded_slash` gate. - `test_rewrite_forward_request_uses_canonical_path_on_the_wire` asserts the fix to the forward-proxy flow. - `parse_l7_config_allow_encoded_slash_{defaults_false,opt_in}` verify the YAML/Rego wiring. --- architecture/sandbox.md | 1 + crates/openshell-policy/src/lib.rs | 8 + .../data/sandbox-policy.rego | 11 +- crates/openshell-sandbox/src/l7/mod.rs | 41 ++ crates/openshell-sandbox/src/l7/path.rs | 563 ++++++++++++++++++ crates/openshell-sandbox/src/l7/relay.rs | 18 +- crates/openshell-sandbox/src/l7/rest.rs | 302 +++++++++- crates/openshell-sandbox/src/proxy.rs | 105 +++- .../tests/websocket_upgrade.rs | 4 +- proto/sandbox.proto | 5 + 10 files changed, 1018 insertions(+), 40 deletions(-) create mode 100644 crates/openshell-sandbox/src/l7/path.rs diff --git a/architecture/sandbox.md b/architecture/sandbox.md index c7e789caeb..fcef69b4fa 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -32,6 +32,7 @@ All paths are relative to `crates/openshell-sandbox/src/`. | `l7/tls.rs` | Ephemeral CA generation (`SandboxCa`), per-hostname leaf cert cache (`CertCache`), TLS termination/connection helpers, `looks_like_tls()` auto-detection | | `l7/relay.rs` | Protocol-aware bidirectional relay with per-request OPA evaluation, credential-injection-only passthrough relay | | `l7/rest.rs` | HTTP/1.1 request/response parsing, body framing (Content-Length, chunked), deny response generation | +| `l7/path.rs` | Request-target canonicalization: percent-decoding, dot-segment resolution, `;params` stripping, encoded-slash policy (opt-in per endpoint via `allow_encoded_slash: true` for upstreams like GitLab that embed `%2F` in paths). Single source of truth for the path both OPA evaluates and the upstream receives. | | `l7/provider.rs` | `L7Provider` trait and `L7Request`/`BodyLength` types | | `secrets.rs` | `SecretResolver` credential placeholder system — placeholder generation, multi-location rewriting (headers, query params, path segments, Basic auth), fail-closed scanning, secret validation, percent-encoding | diff --git a/crates/openshell-policy/src/lib.rs b/crates/openshell-policy/src/lib.rs index 2c8c0cc769..0296879afd 100644 --- a/crates/openshell-policy/src/lib.rs +++ b/crates/openshell-policy/src/lib.rs @@ -109,6 +109,12 @@ struct NetworkEndpointDef { allowed_ips: Vec, #[serde(default, skip_serializing_if = "Vec::is_empty")] deny_rules: Vec, + /// When true, percent-encoded `/` (`%2F`) is preserved in path segments + /// rather than rejected by the L7 path canonicalizer. Required for + /// upstreams like GitLab that embed `%2F` in namespaced resource paths. + /// Defaults to false (strict). + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + allow_encoded_slash: bool, } fn is_zero(v: &u16) -> bool { @@ -261,6 +267,7 @@ fn to_proto(raw: PolicyFile) -> SandboxPolicy { .collect(), }) .collect(), + allow_encoded_slash: e.allow_encoded_slash, } }) .collect(), @@ -400,6 +407,7 @@ fn from_proto(policy: &SandboxPolicy) -> PolicyFile { .collect(), }) .collect(), + allow_encoded_slash: e.allow_encoded_slash, } }) .collect(), diff --git a/crates/openshell-sandbox/data/sandbox-policy.rego b/crates/openshell-sandbox/data/sandbox-policy.rego index 939c2e5cb1..a1a980a9e0 100644 --- a/crates/openshell-sandbox/data/sandbox-policy.rego +++ b/crates/openshell-sandbox/data/sandbox-policy.rego @@ -328,6 +328,13 @@ method_matches(actual, expected) if { } # Path matching: "**" matches everything; otherwise glob.match with "/" delimiter. +# +# INVARIANT: `input.request.path` is canonicalized by the sandbox before +# policy evaluation — percent-decoded, dot-segments resolved, doubled +# slashes collapsed, `;params` stripped, `%2F` rejected (unless an +# endpoint opts in). Patterns here must therefore match canonical paths; +# do not attempt defensive matching against `..` or `%2e%2e` — those +# inputs are rejected at the L7 parser boundary before this rule runs. path_matches(_, "**") if true path_matches(actual, pattern) if { @@ -394,8 +401,8 @@ command_matches(actual, expected) if { # --- Matched endpoint config (for L7 and allowed_ips extraction) --- # Returns the raw endpoint object for the matched policy + host:port. -# Used by Rust to extract L7 config (protocol, tls, enforcement) and/or -# allowed_ips for SSRF allowlist validation. +# Used by Rust to extract L7 config (protocol, tls, enforcement, +# allow_encoded_slash) and/or allowed_ips for SSRF allowlist validation. # Per-policy helper: returns matching endpoint configs for a single policy. _policy_endpoint_configs(policy) := [ep | diff --git a/crates/openshell-sandbox/src/l7/mod.rs b/crates/openshell-sandbox/src/l7/mod.rs index d354f562b2..3b26f8122c 100644 --- a/crates/openshell-sandbox/src/l7/mod.rs +++ b/crates/openshell-sandbox/src/l7/mod.rs @@ -9,6 +9,7 @@ //! evaluated against OPA policy, and either forwarded or denied. pub mod inference; +pub mod path; pub mod provider; pub mod relay; pub mod rest; @@ -59,6 +60,10 @@ pub struct L7EndpointConfig { pub protocol: L7Protocol, pub tls: TlsMode, pub enforcement: EnforcementMode, + /// When true, percent-encoded `/` (`%2F`) is preserved in path segments + /// rather than rejected at the parser. Needed by upstreams like GitLab + /// that embed `%2F` in namespaced project paths. Defaults to false. + pub allow_encoded_slash: bool, } /// Result of an L7 policy decision for a single request. @@ -122,10 +127,13 @@ pub fn parse_l7_config(val: ®orus::Value) -> Option { _ => EnforcementMode::Audit, }; + let allow_encoded_slash = get_object_bool(val, "allow_encoded_slash").unwrap_or(false); + Some(L7EndpointConfig { protocol, tls, enforcement, + allow_encoded_slash, }) } @@ -141,6 +149,19 @@ pub fn parse_tls_mode(val: ®orus::Value) -> TlsMode { } } +/// Extract a bool value from a regorus object. Returns `None` when the key +/// is absent or not a boolean. +fn get_object_bool(val: ®orus::Value, key: &str) -> Option { + let key_val = regorus::Value::String(key.into()); + match val { + regorus::Value::Object(map) => match map.get(&key_val) { + Some(regorus::Value::Bool(b)) => Some(*b), + _ => None, + }, + _ => None, + } +} + /// Extract a string value from a regorus object. fn get_object_str(val: ®orus::Value, key: &str) -> Option { let key_val = regorus::Value::String(key.into()); @@ -746,6 +767,26 @@ mod tests { assert!(parse_l7_config(&val).is_none()); } + #[test] + fn parse_l7_config_allow_encoded_slash_defaults_false() { + let val = regorus::Value::from_json_str( + r#"{"protocol": "rest", "host": "api.example.com", "port": 443}"#, + ) + .unwrap(); + let config = parse_l7_config(&val).unwrap(); + assert!(!config.allow_encoded_slash); + } + + #[test] + fn parse_l7_config_allow_encoded_slash_opt_in() { + let val = regorus::Value::from_json_str( + r#"{"protocol": "rest", "host": "gitlab.example.com", "port": 443, "allow_encoded_slash": true}"#, + ) + .unwrap(); + let config = parse_l7_config(&val).unwrap(); + assert!(config.allow_encoded_slash); + } + #[test] fn validate_rules_and_access_mutual_exclusion() { let data = serde_json::json!({ diff --git a/crates/openshell-sandbox/src/l7/path.rs b/crates/openshell-sandbox/src/l7/path.rs new file mode 100644 index 0000000000..1b425b5f49 --- /dev/null +++ b/crates/openshell-sandbox/src/l7/path.rs @@ -0,0 +1,563 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! HTTP request-target canonicalization for L7 policy enforcement. +//! +//! The L7 REST proxy evaluates OPA rules against the request path and +//! forwards the raw request line to the upstream server. If the path the +//! policy sees is not the path the upstream dispatches on, any path-based +//! allow rule can be bypassed with non-canonical encodings (`..`, `%2e%2e`, +//! `//`, `;params`). This module resolves that divergence by producing a +//! single canonical path that is both the input to policy evaluation and +//! the bytes written onto the wire. +//! +//! Behavior for v1: +//! - Percent-decode unreserved path bytes; preserve the rest as uppercase +//! `%HH`. +//! - Resolve `.` and `..` segments per RFC 3986 Section 5.2.4. `..` that +//! would escape the root is rejected rather than silently clamped to +//! `/` — non-canonical input is almost always adversarial. +//! - Collapse repeated slashes. +//! - Reject control bytes (`0x00..=0x1F`, `0x7F`), fragments in the +//! request-target, raw non-ASCII bytes, and paths that cannot be parsed +//! as origin-form. +//! - Strip trailing `;params` from each segment by default (Tomcat-class +//! `;jsessionid` ACL-bypass mitigation). +//! - Reject `%2F` (encoded slash) inside a segment by default. Operators +//! can opt in per-endpoint for APIs that rely on encoded slashes in +//! slugs. + +use thiserror::Error; + +/// Reasons a request-target can be rejected at the canonicalization boundary. +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum CanonicalizeError { + #[error("request-target contains a null or control byte")] + NullOrControlByte, + #[error("request-target contains an invalid percent-encoded sequence")] + InvalidPercentEncoding, + #[error("request-target contains an encoded '/' (%2F) which is not allowed on this endpoint")] + EncodedSlashNotAllowed, + #[error("request-target contains a fragment")] + FragmentInRequestTarget, + #[error("request-target contains raw non-ASCII bytes; non-ASCII must be percent-encoded")] + NonAscii, + #[error("request-target's `..` segment would escape the path root")] + TraversalAboveRoot, + #[error("request-target exceeds the configured maximum length")] + PathTooLong, + #[error("request-target is not a valid origin-form path")] + MalformedTarget, +} + +/// Options controlling canonicalization strictness. +/// +/// Produced by the endpoint configuration. Defaults are intentionally strict: +/// operators opt in to looser behavior per-endpoint when the upstream API +/// requires it. +#[derive(Debug, Clone, Copy)] +pub struct CanonicalizeOptions { + /// When `true`, `%2F` inside a segment is preserved (re-emitted as + /// `%2F` on the wire) rather than rejected. Defaults to `false`. + pub allow_encoded_slash: bool, + /// When `true`, RFC 3986 path parameters (`;param`) are stripped from + /// each segment before policy evaluation and before forwarding. + /// Defaults to `true`: path parameters are an ambiguity surface + /// historically used to bypass ACLs and are not part of any policy + /// we author. + pub strip_path_parameters: bool, +} + +impl Default for CanonicalizeOptions { + fn default() -> Self { + Self { + allow_encoded_slash: false, + strip_path_parameters: true, + } + } +} + +/// Result of a successful canonicalization. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CanonicalPath { + /// The canonical path. Always starts with `/`. Contains no `.`/`..` + /// segments, no doubled slashes, and no `;params` (when stripping is + /// enabled). + pub path: String, + /// `true` if the canonical form differs from the input. Callers use + /// this to decide whether to rewrite the outbound request line. + pub rewritten: bool, +} + +/// Maximum accepted length of an origin-form path (bytes). +pub(crate) const MAX_PATH_LEN: usize = 4 * 1024; + +/// Sentinel byte used to represent a `%2F`-decoded slash inside a segment. +/// Chosen from the C0 control range so no legitimate decoded byte collides +/// with it; any raw `0x01` in the input is rejected up front. +const ENCODED_SLASH_SENTINEL: u8 = 0x01; + +/// Canonicalize an HTTP request-target's path component. +/// +/// Accepts origin-form (`"/a/b?q=1"`) or absolute-form (`"http://h/a/b"`) +/// targets. Asterisk-form (`"*"`, used only for `OPTIONS *`) is rejected +/// because the L7 enforcement pipeline does not handle it. +/// +/// Returns the canonical path plus the original query suffix (byte-for-byte +/// as supplied by the client). Query-parameter parsing is left to the +/// caller — this function only operates on the path component. +pub fn canonicalize_request_target( + target: &str, + opts: &CanonicalizeOptions, +) -> Result<(CanonicalPath, Option), CanonicalizeError> { + // 1. Reject control bytes and raw non-ASCII outright. These tests also + // cover CR/LF which are never legal in a single-line request-target. + for &b in target.as_bytes() { + if b == 0 || b == b'\n' || b == b'\r' || b == b'\t' || b == 0x7F { + return Err(CanonicalizeError::NullOrControlByte); + } + if b < 0x20 { + return Err(CanonicalizeError::NullOrControlByte); + } + if b >= 0x80 { + return Err(CanonicalizeError::NonAscii); + } + } + + // 2. Reject fragments — forbidden in request-target per RFC 7230. + if target.contains('#') { + return Err(CanonicalizeError::FragmentInRequestTarget); + } + + // 3. Split off query at the first `?`. Query is preserved verbatim. + let (path_part, query_part) = match target.split_once('?') { + Some((p, q)) => (p, Some(q.to_string())), + None => (target, None), + }; + + // 4. Handle absolute-form by stripping scheme://authority. + let raw_path = if let Some(idx) = path_part.find("://") { + let after_scheme = &path_part[idx + 3..]; + match after_scheme.find('/') { + Some(slash) => &after_scheme[slash..], + None => "/", + } + } else { + path_part + }; + + // 5. Empty is equivalent to "/". + let raw_path = if raw_path.is_empty() { "/" } else { raw_path }; + + // 6. Must begin with '/' (origin-form). + if !raw_path.starts_with('/') { + return Err(CanonicalizeError::MalformedTarget); + } + + // 7. Length bound. + if raw_path.len() > MAX_PATH_LEN { + return Err(CanonicalizeError::PathTooLong); + } + + // 8. Percent-decode the path into bytes. `%2F` is replaced by a + // sentinel byte so that subsequent `/` splitting cannot confuse it + // with a real path separator. + let decoded = percent_decode_with_sentinel(raw_path.as_bytes(), opts.allow_encoded_slash)?; + + // 9. Split on literal `/` and resolve dot-segments. + let segments = split_path_segments(&decoded); + let resolved = resolve_dot_segments(segments)?; + + // 10. Reconstruct. Strip `;params` per segment if requested; re-encode + // any byte that must be percent-encoded in the pchar set. + let canonical = build_canonical_path(&resolved, decoded.last().copied() == Some(b'/'), opts); + + let rewritten = canonical != raw_path; + Ok(( + CanonicalPath { + path: canonical, + rewritten, + }, + query_part, + )) +} + +// --------------------------------------------------------------------------- +// Internals +// --------------------------------------------------------------------------- + +fn percent_decode_with_sentinel( + raw: &[u8], + allow_encoded_slash: bool, +) -> Result, CanonicalizeError> { + let mut out = Vec::with_capacity(raw.len()); + let mut i = 0; + while i < raw.len() { + let b = raw[i]; + if b == ENCODED_SLASH_SENTINEL { + // Raw sentinel byte in input — already rejected by the C0 + // control-byte sweep above, but double-check here to avoid + // collisions in case the sweep is ever relaxed. + return Err(CanonicalizeError::NullOrControlByte); + } + if b == b'%' { + if i + 2 >= raw.len() { + return Err(CanonicalizeError::InvalidPercentEncoding); + } + let decoded = match (decode_hex(raw[i + 1]), decode_hex(raw[i + 2])) { + (Some(hi), Some(lo)) => (hi << 4) | lo, + _ => return Err(CanonicalizeError::InvalidPercentEncoding), + }; + if decoded == b'/' { + if !allow_encoded_slash { + return Err(CanonicalizeError::EncodedSlashNotAllowed); + } + out.push(ENCODED_SLASH_SENTINEL); + } else if decoded == 0 || decoded == 0x7F || (decoded < 0x20 && decoded != b'\t') { + return Err(CanonicalizeError::NullOrControlByte); + } else if decoded == b'\n' || decoded == b'\r' || decoded == b'\t' { + // %-encoded CR/LF/TAB are still control bytes; reject. + return Err(CanonicalizeError::NullOrControlByte); + } else { + out.push(decoded); + } + i += 3; + } else { + out.push(b); + i += 1; + } + } + Ok(out) +} + +fn split_path_segments(decoded: &[u8]) -> Vec<&[u8]> { + // decoded is guaranteed to start with `/`. Skip the leading `/` and + // split on subsequent `/` bytes. The sentinel byte for encoded slashes + // never matches, so it stays inside its segment. + decoded[1..].split(|&b| b == b'/').collect() +} + +fn resolve_dot_segments(segments: Vec<&[u8]>) -> Result>, CanonicalizeError> { + let mut stack: Vec> = Vec::with_capacity(segments.len()); + let last = segments.len().saturating_sub(1); + for (idx, seg) in segments.into_iter().enumerate() { + if seg == b".." { + if stack.pop().is_none() { + return Err(CanonicalizeError::TraversalAboveRoot); + } + if idx == last { + // A trailing `..` leaves a "directory" (trailing slash). + stack.push(Vec::new()); + } + continue; + } + if seg == b"." { + if idx == last { + stack.push(Vec::new()); + } + continue; + } + if seg.is_empty() && idx != last { + // Collapse repeated slashes except at the very end, where an + // empty trailing segment encodes a trailing `/`. + continue; + } + stack.push(seg.to_vec()); + } + Ok(stack) +} + +fn build_canonical_path( + segments: &[Vec], + _trailing_slash_hint: bool, + opts: &CanonicalizeOptions, +) -> String { + let mut out = String::from("/"); + for (idx, seg) in segments.iter().enumerate() { + if idx > 0 { + out.push('/'); + } + let trimmed: &[u8] = if opts.strip_path_parameters { + match seg.iter().position(|&b| b == b';') { + Some(pos) => &seg[..pos], + None => seg, + } + } else { + seg + }; + for &b in trimmed { + if b == ENCODED_SLASH_SENTINEL { + out.push_str("%2F"); + } else if is_pchar_unreserved(b) { + out.push(b as char); + } else { + out.push('%'); + out.push(upper_hex_nibble(b >> 4)); + out.push(upper_hex_nibble(b & 0x0F)); + } + } + } + out +} + +fn is_pchar_unreserved(b: u8) -> bool { + // RFC 3986 pchar without the percent-encoded slot — i.e. bytes we emit + // literally. Unreserved plus RFC 3986 sub-delims plus `:` and `@`. + b.is_ascii_alphanumeric() + || matches!( + b, + b'-' | b'.' + | b'_' + | b'~' + | b'!' + | b'$' + | b'&' + | b'\'' + | b'(' + | b')' + | b'*' + | b'+' + | b',' + | b';' + | b'=' + | b':' + | b'@' + ) +} + +fn decode_hex(b: u8) -> Option { + match b { + b'0'..=b'9' => Some(b - b'0'), + b'a'..=b'f' => Some(b - b'a' + 10), + b'A'..=b'F' => Some(b - b'A' + 10), + _ => None, + } +} + +fn upper_hex_nibble(n: u8) -> char { + match n { + 0..=9 => (b'0' + n) as char, + 10..=15 => (b'A' + (n - 10)) as char, + _ => unreachable!("nibble out of range"), + } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + fn canon(input: &str) -> Result { + let opts = CanonicalizeOptions::default(); + canonicalize_request_target(input, &opts).map(|(p, _)| p.path) + } + + fn canon_with(input: &str, opts: CanonicalizeOptions) -> Result { + canonicalize_request_target(input, &opts).map(|(p, _)| p.path) + } + + #[test] + fn literal_dot_segments_resolve() { + assert_eq!(canon("/a/./b").unwrap(), "/a/b"); + assert_eq!(canon("/a/b/.").unwrap(), "/a/b/"); + assert_eq!(canon("/a/../b").unwrap(), "/b"); + assert_eq!(canon("/a/b/..").unwrap(), "/a/"); + } + + #[test] + fn percent_encoded_dot_segments_resolve_the_same_way() { + assert_eq!(canon("/public/%2e%2e/secret").unwrap(), "/secret"); + assert_eq!(canon("/public/%2E%2E/secret").unwrap(), "/secret"); + assert_eq!(canon("/public/%2e/secret").unwrap(), "/public/secret"); + } + + #[test] + fn traversal_above_root_is_rejected() { + assert_eq!(canon("/.."), Err(CanonicalizeError::TraversalAboveRoot)); + assert_eq!( + canon("/a/../.."), + Err(CanonicalizeError::TraversalAboveRoot) + ); + assert_eq!( + canon("/a/%2e%2e/%2e%2e"), + Err(CanonicalizeError::TraversalAboveRoot) + ); + } + + #[test] + fn doubled_slashes_collapse() { + assert_eq!(canon("//").unwrap(), "/"); + assert_eq!(canon("//public//../secret").unwrap(), "/secret"); + assert_eq!(canon("/public//secret").unwrap(), "/public/secret"); + } + + #[test] + fn encoded_slash_rejected_by_default() { + assert_eq!( + canon("/a/%2f/b"), + Err(CanonicalizeError::EncodedSlashNotAllowed) + ); + assert_eq!( + canon("/public/..%2fsecret"), + Err(CanonicalizeError::EncodedSlashNotAllowed) + ); + } + + #[test] + fn encoded_slash_preserved_when_opted_in() { + let opts = CanonicalizeOptions { + allow_encoded_slash: true, + ..CanonicalizeOptions::default() + }; + assert_eq!(canon_with("/a/%2f/b", opts).unwrap(), "/a/%2F/b"); + assert_eq!(canon_with("/a/%2F/b", opts).unwrap(), "/a/%2F/b"); + } + + #[test] + fn null_and_control_bytes_rejected() { + assert_eq!(canon("/a%00b"), Err(CanonicalizeError::NullOrControlByte)); + assert_eq!(canon("/a%0Ab"), Err(CanonicalizeError::NullOrControlByte)); + assert_eq!(canon("/a%0Db"), Err(CanonicalizeError::NullOrControlByte)); + assert_eq!(canon("/a%7Fb"), Err(CanonicalizeError::NullOrControlByte)); + // Raw CR/LF/TAB in input should also fail. Build strings via + // byte-level concatenation since the literals in the source are + // otherwise flagged as control bytes in CI. + let mut raw = String::from("/a"); + raw.push('\n'); + raw.push('b'); + assert_eq!(canon(&raw), Err(CanonicalizeError::NullOrControlByte)); + } + + #[test] + fn fragment_rejected() { + assert_eq!( + canon("/a#b"), + Err(CanonicalizeError::FragmentInRequestTarget) + ); + } + + #[test] + fn absolute_form_strips_authority() { + assert_eq!(canon("http://host/a/../b").unwrap(), "/b"); + assert_eq!(canon("https://host").unwrap(), "/"); + assert_eq!(canon("http://host:443/foo").unwrap(), "/foo"); + } + + #[test] + fn legitimate_percent_encoded_bytes_round_trip() { + assert_eq!( + canon("/files/hello%20world.txt").unwrap(), + "/files/hello%20world.txt" + ); + assert_eq!(canon("/search/a%3Fb").unwrap(), "/search/a%3Fb"); + assert_eq!(canon("/users/%40alice").unwrap(), "/users/@alice"); + } + + #[test] + fn path_parameters_stripped_by_default() { + assert_eq!(canon("/a;jsessionid=xyz/b").unwrap(), "/a/b"); + assert_eq!(canon("/public;x=1/../secret").unwrap(), "/secret"); + } + + #[test] + fn path_parameters_preserved_when_disabled() { + let opts = CanonicalizeOptions { + strip_path_parameters: false, + ..CanonicalizeOptions::default() + }; + assert_eq!( + canon_with("/a;jsessionid=xyz/b", opts).unwrap(), + "/a;jsessionid=xyz/b" + ); + } + + #[test] + fn non_ascii_raw_byte_rejected() { + let mut raw = String::from("/a"); + raw.push('é'); + assert_eq!(canon(&raw), Err(CanonicalizeError::NonAscii)); + } + + #[test] + fn percent_encoded_non_ascii_bytes_round_trip() { + // `é` in UTF-8 is 0xC3 0xA9. The proxy treats the path as opaque + // bytes; percent-encoded high bytes pass through unchanged. + assert_eq!(canon("/users/caf%C3%A9").unwrap(), "/users/caf%C3%A9"); + } + + #[test] + fn empty_and_root_equivalent() { + assert_eq!(canon("").unwrap(), "/"); + assert_eq!(canon("/").unwrap(), "/"); + } + + #[test] + fn path_too_long_rejected() { + let long = format!("/{}", "a".repeat(MAX_PATH_LEN)); + assert_eq!(canon(&long), Err(CanonicalizeError::PathTooLong)); + } + + #[test] + fn mixed_case_percent_normalizes_to_uppercase() { + // Request comes in with lowercase %c3 — after canonicalization we + // emit %C3 so policy authors don't need to enumerate both cases. + assert_eq!(canon("/a/caf%c3%a9").unwrap(), "/a/caf%C3%A9"); + } + + #[test] + fn rewritten_flag_reflects_transformation() { + let (canon, _) = + canonicalize_request_target("/a", &CanonicalizeOptions::default()).unwrap(); + assert!(!canon.rewritten); + let (canon, _) = + canonicalize_request_target("/a/../b", &CanonicalizeOptions::default()).unwrap(); + assert!(canon.rewritten); + } + + #[test] + fn query_suffix_is_returned_separately() { + let (canon, query) = + canonicalize_request_target("/a?q=1&r=2", &CanonicalizeOptions::default()).unwrap(); + assert_eq!(canon.path, "/a"); + assert_eq!(query.as_deref(), Some("q=1&r=2")); + } + + // --------------------------------------------------------------------- + // Regression tests for the documented attack payloads. Every one of + // these used to bypass a `/public/**` allow rule because the proxy and + // the OPA policy never agreed with the upstream on what path was being + // dispatched. + // --------------------------------------------------------------------- + + #[test] + fn regression_public_slash_dotdot_secret() { + assert_eq!(canon("/public/../secret").unwrap(), "/secret"); + } + + #[test] + fn regression_public_slash_percent_dotdot_secret() { + assert_eq!(canon("/public/%2e%2e/secret").unwrap(), "/secret"); + assert_eq!(canon("/public/%2E%2E/secret").unwrap(), "/secret"); + } + + #[test] + fn regression_percent_encoded_slash_in_dotdot_rejected() { + assert_eq!( + canon("/public/%2E%2E%2Fsecret"), + Err(CanonicalizeError::EncodedSlashNotAllowed) + ); + } + + #[test] + fn regression_double_slash_prefix() { + assert_eq!(canon("//public/../secret").unwrap(), "/secret"); + } + + #[test] + fn regression_dot_slash_dotdot() { + assert_eq!(canon("/public/./../secret").unwrap(), "/secret"); + } +} diff --git a/crates/openshell-sandbox/src/l7/relay.rs b/crates/openshell-sandbox/src/l7/relay.rs index 110f777e91..a0e54062d9 100644 --- a/crates/openshell-sandbox/src/l7/relay.rs +++ b/crates/openshell-sandbox/src/l7/relay.rs @@ -128,9 +128,17 @@ where C: AsyncRead + AsyncWrite + Unpin + Send, U: AsyncRead + AsyncWrite + Unpin + Send, { + // Build a provider carrying the per-endpoint canonicalization options so + // request parsing honors the endpoint's `allow_encoded_slash` setting + // (e.g. APIs like GitLab that embed `%2F` in path segments). + let provider = + crate::l7::rest::RestProvider::with_options(crate::l7::path::CanonicalizeOptions { + allow_encoded_slash: config.allow_encoded_slash, + ..Default::default() + }); loop { // Parse one HTTP request from client - let req = match crate::l7::rest::RestProvider.parse_request(client).await { + let req = match provider.parse_request(client).await { Ok(Some(req)) => req, Ok(None) => return Ok(()), // Client closed connection Err(e) => { @@ -274,7 +282,7 @@ where } } else { // Enforce mode: deny with 403 and close connection (use redacted target) - crate::l7::rest::RestProvider + provider .deny_with_redacted_target( &req, &ctx.policy_name, @@ -374,7 +382,11 @@ where C: AsyncRead + AsyncWrite + Unpin + Send, U: AsyncRead + AsyncWrite + Unpin + Send, { - let provider = crate::l7::rest::RestProvider; + // Passthrough path: no L7 policy is enforced here, so use default + // (strict) canonicalization options. Calls to GitLab-style APIs that + // need `%2F` must be configured as L7 endpoints so the per-endpoint + // `allow_encoded_slash` opt-in applies. + let provider = crate::l7::rest::RestProvider::default(); let mut request_count: u64 = 0; let resolver = ctx.secret_resolver.as_deref(); diff --git a/crates/openshell-sandbox/src/l7/rest.rs b/crates/openshell-sandbox/src/l7/rest.rs index 6bbf7be4ee..4d8909b9eb 100644 --- a/crates/openshell-sandbox/src/l7/rest.rs +++ b/crates/openshell-sandbox/src/l7/rest.rs @@ -22,14 +22,33 @@ const RELAY_BUF_SIZE: usize = 8192; const RELAY_EOF_IDLE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); /// HTTP/1.1 REST protocol provider. -pub struct RestProvider; +/// +/// Carries the path-canonicalization options derived from the endpoint +/// config so that different endpoints (e.g. one backed by GitLab that needs +/// `%2F` in paths and one backed by a strict API) can apply different +/// canonicalization strictness to the same `RestProvider` call surface. +#[derive(Debug, Clone, Default)] +pub struct RestProvider { + canonicalize_options: crate::l7::path::CanonicalizeOptions, +} + +impl RestProvider { + /// Construct a provider with explicit canonicalization options. Used by + /// `relay_rest` so endpoint config can opt in to looser behavior such + /// as `allow_encoded_slash`. + pub fn with_options(canonicalize_options: crate::l7::path::CanonicalizeOptions) -> Self { + Self { + canonicalize_options, + } + } +} impl L7Provider for RestProvider { async fn parse_request( &self, client: &mut C, ) -> Result> { - parse_http_request(client).await + parse_http_request(client, &self.canonicalize_options).await } async fn relay( @@ -78,7 +97,10 @@ impl RestProvider { /// forwarded upstream without L7 policy evaluation -- a request /// smuggling vulnerability. Byte-at-a-time overhead is negligible for /// the typical 200-800 byte headers on L7-inspected REST endpoints. -async fn parse_http_request(client: &mut C) -> Result> { +async fn parse_http_request( + client: &mut C, + canonicalize_options: &crate::l7::path::CanonicalizeOptions, +) -> Result> { let mut buf = Vec::with_capacity(4096); loop { @@ -149,17 +171,68 @@ async fn parse_http_request(client: &mut C) -> Result parse_query_params(q)?, + None => HashMap::new(), + }; + + if canonical.rewritten { + buf = rewrite_request_line_target( + &buf, + &method, + &canonical.path, + raw_query.as_deref(), + version, + )?; + } Ok(Some(L7Request { action: method, - target: path, + target: canonical.path, query_params, raw_header: buf, // exact header bytes up to and including \r\n\r\n body_length, })) } +/// Rebuild the request line in a raw HTTP header block with a canonicalized +/// target. Called when the canonical path differs from what the client sent, +/// so the upstream dispatches on the exact bytes the policy engine evaluated. +fn rewrite_request_line_target( + raw: &[u8], + method: &str, + canonical_path: &str, + raw_query: Option<&str>, + version: &str, +) -> Result> { + let eol = raw + .windows(2) + .position(|w| w == b"\r\n") + .ok_or_else(|| miette!("request line missing CRLF"))?; + let rest = &raw[eol..]; + let new_target = match raw_query { + Some(q) if !q.is_empty() => format!("{canonical_path}?{q}"), + _ => canonical_path.to_string(), + }; + let new_request_line = format!("{method} {new_target} {version}"); + let mut out = Vec::with_capacity(new_request_line.len() + rest.len()); + out.extend_from_slice(new_request_line.as_bytes()); + out.extend_from_slice(rest); + Ok(out) +} + pub(crate) fn parse_target_query(target: &str) -> Result<(String, HashMap>)> { match target.split_once('?') { Some((path, query)) => Ok((path.to_string(), parse_query_params(query)?)), @@ -167,7 +240,7 @@ pub(crate) fn parse_target_query(target: &str) -> Result<(String, HashMap Result>> { +pub(crate) fn parse_query_params(query: &str) -> Result>> { let mut params: HashMap> = HashMap::new(); if query.is_empty() { return Ok(params); @@ -1064,7 +1137,11 @@ mod tests { .await .unwrap(); }); - let result = parse_http_request(&mut client).await; + let result = parse_http_request( + &mut client, + &crate::l7::path::CanonicalizeOptions::default(), + ) + .await; assert!(result.is_err(), "Must reject headers with bare LF"); } @@ -1077,7 +1154,11 @@ mod tests { raw.extend_from_slice(b"GET /api HTTP/1.1\r\nHost: x\r\nX-Bad: \xc0\xaf\r\n\r\n"); writer.write_all(&raw).await.unwrap(); }); - let result = parse_http_request(&mut client).await; + let result = parse_http_request( + &mut client, + &crate::l7::path::CanonicalizeOptions::default(), + ) + .await; assert!(result.is_err(), "Must reject headers with invalid UTF-8"); } @@ -1091,10 +1172,180 @@ mod tests { .await .unwrap(); }); - let result = parse_http_request(&mut client).await; + let result = parse_http_request( + &mut client, + &crate::l7::path::CanonicalizeOptions::default(), + ) + .await; assert!(result.is_err(), "Must reject unsupported HTTP version"); } + #[tokio::test] + async fn parse_http_request_canonicalizes_target_and_rewrites_raw_header() { + let (mut client, mut writer) = tokio::io::duplex(4096); + tokio::spawn(async move { + writer + .write_all(b"GET /public/../secret HTTP/1.1\r\nHost: api.example.com\r\n\r\n") + .await + .unwrap(); + }); + let req = parse_http_request( + &mut client, + &crate::l7::path::CanonicalizeOptions::default(), + ) + .await + .expect("request should parse") + .expect("request should exist"); + // Path fed to OPA evaluation is canonical. + assert_eq!(req.target, "/secret"); + // raw_header (forwarded byte-for-byte to upstream) is also canonical + // — this is the invariant the L7 canonicalization PR must uphold. + assert_eq!( + req.raw_header, b"GET /secret HTTP/1.1\r\nHost: api.example.com\r\n\r\n", + "outbound request line must carry the canonical path" + ); + } + + #[tokio::test] + async fn parse_http_request_canonicalization_preserves_query_string() { + let (mut client, mut writer) = tokio::io::duplex(4096); + tokio::spawn(async move { + writer + .write_all(b"GET /public/../v1/list?limit=10&sort=asc HTTP/1.1\r\nHost: h\r\n\r\n") + .await + .unwrap(); + }); + let req = parse_http_request( + &mut client, + &crate::l7::path::CanonicalizeOptions::default(), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(req.target, "/v1/list"); + assert_eq!( + req.raw_header, b"GET /v1/list?limit=10&sort=asc HTTP/1.1\r\nHost: h\r\n\r\n", + "canonical rewrite must preserve the query string verbatim" + ); + } + + #[tokio::test] + async fn parse_http_request_leaves_canonical_input_byte_for_byte() { + // When the input is already canonical, the raw_header must pass + // through unchanged — otherwise legitimate traffic pays a rewrite + // cost on every request. + let (mut client, mut writer) = tokio::io::duplex(4096); + tokio::spawn(async move { + writer + .write_all(b"GET /api/v1/users HTTP/1.1\r\nHost: api.example.com\r\n\r\n") + .await + .unwrap(); + }); + let req = parse_http_request( + &mut client, + &crate::l7::path::CanonicalizeOptions::default(), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(req.target, "/api/v1/users"); + assert_eq!( + req.raw_header, + b"GET /api/v1/users HTTP/1.1\r\nHost: api.example.com\r\n\r\n", + ); + } + + #[tokio::test] + async fn parse_http_request_rejects_traversal_above_root() { + let (mut client, mut writer) = tokio::io::duplex(4096); + tokio::spawn(async move { + writer + .write_all(b"GET /.. HTTP/1.1\r\nHost: h\r\n\r\n") + .await + .unwrap(); + }); + let result = parse_http_request( + &mut client, + &crate::l7::path::CanonicalizeOptions::default(), + ) + .await; + assert!( + result.is_err(), + "a target that escapes the path root must be rejected at the parser" + ); + } + + #[tokio::test] + async fn parse_http_request_accepts_encoded_slash_when_endpoint_opts_in() { + // GitLab-style endpoints legitimately embed `%2F` in path segments + // (e.g. `/api/v4/projects/group%2Fproject`). Passing a provider + // constructed with `allow_encoded_slash: true` models the + // endpoint-config wiring that flows from `L7EndpointConfig`. + let (mut client, mut writer) = tokio::io::duplex(4096); + tokio::spawn(async move { + writer + .write_all(b"GET /api/v4/projects/group%2Fproject HTTP/1.1\r\nHost: g\r\n\r\n") + .await + .unwrap(); + }); + let options = crate::l7::path::CanonicalizeOptions { + allow_encoded_slash: true, + ..Default::default() + }; + let req = parse_http_request(&mut client, &options) + .await + .unwrap() + .unwrap(); + assert_eq!(req.target, "/api/v4/projects/group%2Fproject"); + } + + #[tokio::test] + async fn parse_http_request_rejects_encoded_slash_by_default() { + // Default strict options must reject `%2F` — this is the security + // posture for endpoints where an encoded slash would let an + // attacker disagree with the upstream on segment boundaries. + let (mut client, mut writer) = tokio::io::duplex(4096); + tokio::spawn(async move { + writer + .write_all(b"GET /api/v4/projects/group%2Fproject HTTP/1.1\r\nHost: g\r\n\r\n") + .await + .unwrap(); + }); + let result = parse_http_request( + &mut client, + &crate::l7::path::CanonicalizeOptions::default(), + ) + .await; + assert!( + result.is_err(), + "default options must reject encoded slashes in the path" + ); + } + + #[tokio::test] + async fn parse_http_request_preserves_http_10_version_on_rewrite() { + let (mut client, mut writer) = tokio::io::duplex(4096); + tokio::spawn(async move { + writer + .write_all(b"GET /a/./b HTTP/1.0\r\nHost: h\r\n\r\n") + .await + .unwrap(); + }); + let req = parse_http_request( + &mut client, + &crate::l7::path::CanonicalizeOptions::default(), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(req.target, "/a/b"); + assert!( + req.raw_header.starts_with(b"GET /a/b HTTP/1.0\r\n"), + "rewrite must preserve the original HTTP version, got: {:?}", + String::from_utf8_lossy(&req.raw_header) + ); + } + #[tokio::test] async fn parse_http_request_splits_path_and_query_params() { let (mut client, mut writer) = tokio::io::duplex(4096); @@ -1106,10 +1357,13 @@ mod tests { .await .unwrap(); }); - let req = parse_http_request(&mut client) - .await - .expect("request should parse") - .expect("request should exist"); + let req = parse_http_request( + &mut client, + &crate::l7::path::CanonicalizeOptions::default(), + ) + .await + .expect("request should parse") + .expect("request should exist"); assert_eq!(req.target, "/download"); assert_eq!( req.query_params.get("slug").cloned(), @@ -1139,10 +1393,13 @@ mod tests { .unwrap(); }); - let first = parse_http_request(&mut client) - .await - .expect("first request should parse") - .expect("expected first request"); + let first = parse_http_request( + &mut client, + &crate::l7::path::CanonicalizeOptions::default(), + ) + .await + .expect("first request should parse") + .expect("expected first request"); assert_eq!(first.action, "GET"); assert_eq!(first.target, "/allowed"); assert!(first.query_params.is_empty()); @@ -1151,10 +1408,13 @@ mod tests { "raw_header must contain only the first request's headers" ); - let second = parse_http_request(&mut client) - .await - .expect("second request should parse") - .expect("expected second request"); + let second = parse_http_request( + &mut client, + &crate::l7::path::CanonicalizeOptions::default(), + ) + .await + .expect("second request should parse") + .expect("expected second request"); assert_eq!(second.action, "POST"); assert_eq!(second.target, "/blocked"); assert!(second.query_params.is_empty()); diff --git a/crates/openshell-sandbox/src/proxy.rs b/crates/openshell-sandbox/src/proxy.rs index 01ff303477..e3d4b5c136 100644 --- a/crates/openshell-sandbox/src/proxy.rs +++ b/crates/openshell-sandbox/src/proxy.rs @@ -1798,15 +1798,21 @@ fn query_allowed_ips( } } +/// Canonicalize the request-target for inference pattern detection. +/// +/// Falls back to the raw path on canonicalization error: the request is then +/// routed through the normal forward path, where `rest.rs::parse_http_request` +/// will reject it properly. Returning the raw path here prevents a crafted +/// target from bypassing inference routing without our detection logic having +/// to implement a second, duplicate error-response surface. fn normalize_inference_path(path: &str) -> String { - if let Some(scheme_idx) = path.find("://") { - let after_scheme = &path[scheme_idx + 3..]; - if let Some(path_start) = after_scheme.find('/') { - return after_scheme[path_start..].to_string(); - } - return "/".to_string(); + match crate::l7::path::canonicalize_request_target( + path, + &crate::l7::path::CanonicalizeOptions::default(), + ) { + Ok((canon, _)) => canon.path, + Err(_) => path.to_string(), } - path.to_string() } /// Extract the hostname from an absolute-form URI used in plain HTTP proxy requests. @@ -2037,8 +2043,11 @@ async fn handle_forward_proxy( secret_resolver: Option>, denial_tx: Option<&mpsc::UnboundedSender>, ) -> Result<()> { - // 1. Parse the absolute-form URI - let (scheme, host, port, path) = match parse_proxy_uri(target_uri) { + // 1. Parse the absolute-form URI. `path` is marked `mut` so that, when an + // L7 config applies, the canonicalized form produced below replaces it + // in-place — keeping OPA evaluation and the bytes written onto the wire + // in sync. See the L7 block below. + let (scheme, host, port, mut path) = match parse_proxy_uri(target_uri) { Ok(parsed) => parsed, Err(e) => { let event = HttpActivityBuilder::new(crate::ocsf_ctx()) @@ -2217,11 +2226,54 @@ async fn handle_forward_proxy( secret_resolver: secret_resolver.clone(), }; - let (target_path, query_params) = crate::l7::rest::parse_target_query(&path) - .unwrap_or_else(|_| (path.clone(), std::collections::HashMap::new())); + // Canonicalize the request-target. The canonical form is fed to OPA + // AND reassigned to the outer `path` variable so the later call to + // `rewrite_forward_request` writes canonical bytes to the upstream. + // This closes the policy/upstream parser-differential at this site; + // without this reassignment, OPA would evaluate the canonical form + // while the upstream re-normalizes the raw input and dispatches on a + // potentially different path. + let canonicalize_options = crate::l7::path::CanonicalizeOptions { + allow_encoded_slash: l7_config.allow_encoded_slash, + ..Default::default() + }; + let query_params = + match crate::l7::path::canonicalize_request_target(&path, &canonicalize_options) { + Ok((canon, query)) => { + let params = match query.as_deref() { + Some(q) => crate::l7::rest::parse_query_params(q).unwrap_or_default(), + None => std::collections::HashMap::new(), + }; + path = canon.path; + params + } + Err(e) => { + let event = NetworkActivityBuilder::new(crate::ocsf_ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .dst_endpoint(Endpoint::from_domain(&host_lc, port)) + .message(format!( + "FORWARD_L7 rejecting non-canonical request-target: {e}" + )) + .build(); + ocsf_emit!(event); + respond( + client, + &build_json_error_response( + 400, + "Bad Request", + "invalid_request_target", + "request-target must be canonical", + ), + ) + .await?; + return Ok(()); + } + }; let request_info = crate::l7::L7RequestInfo { action: method.to_string(), - target: target_path, + target: path.clone(), query_params, }; @@ -3516,6 +3568,35 @@ mod tests { assert!(!result_str.contains("Via: 1.1 openshell-sandbox")); } + #[test] + fn test_rewrite_forward_request_uses_canonical_path_on_the_wire() { + // Regression: the forward-proxy caller must canonicalize first and + // then pass the canonical form to rewrite_forward_request so that + // OPA's policy evaluation and the bytes dispatched to the upstream + // agree. Prior to this guarantee, OPA saw the canonical form while + // the upstream re-normalized the raw path independently, re-opening + // the parser-differential this PR closes. + let raw = b"GET http://host/public/../secret HTTP/1.1\r\nHost: host\r\n\r\n"; + let (canon, _) = crate::l7::path::canonicalize_request_target( + "/public/../secret", + &crate::l7::path::CanonicalizeOptions::default(), + ) + .expect("canonicalization should succeed for the attack payload"); + assert_eq!(canon.path, "/secret"); + + let rewritten = rewrite_forward_request(raw, raw.len(), &canon.path, None) + .expect("rewrite_forward_request should succeed"); + let rewritten_str = String::from_utf8_lossy(&rewritten); + assert!( + rewritten_str.starts_with("GET /secret HTTP/1.1\r\n"), + "outbound request line must use canonical path, got: {rewritten_str:?}" + ); + assert!( + !rewritten_str.contains(".."), + "outbound bytes must not leak the pre-canonical form, got: {rewritten_str:?}" + ); + } + #[test] fn test_rewrite_resolves_placeholder_auth_headers() { let (_, resolver) = SecretResolver::from_provider_env( diff --git a/crates/openshell-sandbox/tests/websocket_upgrade.rs b/crates/openshell-sandbox/tests/websocket_upgrade.rs index ec226c9cf6..81eb46e1f2 100644 --- a/crates/openshell-sandbox/tests/websocket_upgrade.rs +++ b/crates/openshell-sandbox/tests/websocket_upgrade.rs @@ -111,7 +111,7 @@ async fn websocket_upgrade_through_l7_relay_exchanges_message() { // Run the relay in a background task (simulates what relay_rest does) let relay_handle = tokio::spawn(async move { - let outcome = RestProvider + let outcome = RestProvider::default() .relay(&req, &mut client_proxy, &mut upstream) .await .expect("relay should succeed"); @@ -239,7 +239,7 @@ async fn normal_http_request_still_works_after_relay_changes() { let outcome = tokio::time::timeout( std::time::Duration::from_secs(5), - RestProvider.relay(&req, &mut client_proxy, &mut upstream), + RestProvider::default().relay(&req, &mut client_proxy, &mut upstream), ) .await .expect("should not deadlock") diff --git a/proto/sandbox.proto b/proto/sandbox.proto index 974ef91b7f..8a9abbce27 100644 --- a/proto/sandbox.proto +++ b/proto/sandbox.proto @@ -97,6 +97,11 @@ message NetworkEndpoint { // are blocked even if they match an allow rule or access preset. // Deny rules take precedence over allow rules. repeated L7DenyRule deny_rules = 10; + // When true, percent-encoded '/' (%2F) is preserved in path segments + // rather than rejected by the L7 path canonicalizer. Required for + // upstreams like GitLab that embed %2F in namespaced resource paths. + // Defaults to false (strict). + bool allow_encoded_slash = 11; } // An L7 deny rule that blocks specific requests. From a6d45528c11805616e2e1120d2ed2f817fb112fc Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Tue, 21 Apr 2026 08:38:18 -0700 Subject: [PATCH 019/142] feat(server,sandbox): supervisor-initiated SSH connect and exec over gRPC-multiplexed relay (#867) --- architecture/gateway-security.md | 46 +- architecture/gateway.md | 359 +++-- architecture/sandbox-connect.md | 671 ++++----- architecture/sandbox.md | 143 +- architecture/system-architecture.md | 40 +- crates/openshell-cli/src/ssh.rs | 12 +- .../tests/ensure_providers_integration.rs | 35 +- .../openshell-cli/tests/mtls_integration.rs | 21 + .../tests/provider_commands_integration.rs | 35 +- .../sandbox_create_lifecycle_integration.rs | 36 +- .../sandbox_name_fallback_integration.rs | 33 +- crates/openshell-core/src/config.rs | 27 +- .../openshell-driver-kubernetes/src/config.rs | 3 +- .../openshell-driver-kubernetes/src/driver.rs | 95 +- .../openshell-driver-kubernetes/src/grpc.rs | 18 +- .../openshell-driver-kubernetes/src/main.rs | 11 +- crates/openshell-driver-vm/src/driver.rs | 90 +- crates/openshell-sandbox/Cargo.toml | 3 + crates/openshell-sandbox/src/grpc_client.rs | 11 +- crates/openshell-sandbox/src/lib.rs | 33 +- crates/openshell-sandbox/src/main.rs | 10 +- crates/openshell-sandbox/src/ssh.rs | 363 +---- .../src/supervisor_session.rs | 570 ++++++++ crates/openshell-server/src/cli.rs | 5 - crates/openshell-server/src/compute/mod.rs | 107 +- crates/openshell-server/src/grpc/mod.rs | 38 +- crates/openshell-server/src/grpc/sandbox.rs | 451 ++---- crates/openshell-server/src/lib.rs | 15 +- crates/openshell-server/src/multiplex.rs | 12 +- crates/openshell-server/src/ssh_tunnel.rs | 324 ++--- .../src/supervisor_session.rs | 1262 +++++++++++++++++ .../tests/auth_endpoint_integration.rs | 21 + .../tests/edge_tunnel_auth.rs | 34 +- .../tests/multiplex_integration.rs | 34 +- .../tests/multiplex_tls_integration.rs | 34 +- .../tests/supervisor_relay_integration.rs | 572 ++++++++ .../tests/ws_tunnel_integration.rs | 34 +- docs/observability/logging.mdx | 6 +- docs/security/best-practices.mdx | 2 +- proto/compute_driver.proto | 25 - proto/openshell.proto | 120 ++ 41 files changed, 3960 insertions(+), 1801 deletions(-) create mode 100644 crates/openshell-sandbox/src/supervisor_session.rs create mode 100644 crates/openshell-server/src/supervisor_session.rs create mode 100644 crates/openshell-server/tests/supervisor_relay_integration.rs diff --git a/architecture/gateway-security.md b/architecture/gateway-security.md index 319800c081..6baaee88bf 100644 --- a/architecture/gateway-security.md +++ b/architecture/gateway-security.md @@ -269,20 +269,28 @@ The gateway enforces two concurrent connection limits to bound the impact of cre These limits are tracked in-memory and decremented when tunnels close. Exceeding either limit returns HTTP 429 (Too Many Requests). -### NSSH1 Handshake +### Supervisor-Initiated Relay Model -After the gateway connects to the sandbox pod's SSH port, it performs a cryptographic handshake: +The gateway never dials the sandbox. Instead, the sandbox supervisor opens an outbound `ConnectSupervisor` bidirectional gRPC stream to the gateway on startup and keeps it alive for the sandbox lifetime. SSH traffic for `/connect/ssh` (and exec traffic for `ExecSandbox`) rides this same TCP+TLS+HTTP/2 connection as separate multiplexed HTTP/2 streams. The gateway-side registry and `RelayStream` handler live in `crates/openshell-server/src/supervisor_session.rs`; the supervisor-side bridge lives in `crates/openshell-sandbox/src/supervisor_session.rs`. -``` -NSSH1 \n -``` +Per-connection flow: + +1. CLI presents `x-sandbox-id` + `x-sandbox-token` at `/connect/ssh` and passes gateway token validation. +2. Gateway calls `SupervisorSessionRegistry::open_relay(sandbox_id, ...)`, which allocates a `channel_id` (UUID) and sends a `RelayOpen` message to the supervisor over the already-established `ConnectSupervisor` stream. If no session is registered yet, it polls with exponential backoff up to a bounded timeout (30 s for `/connect/ssh`, 15 s for `ExecSandbox`). +3. The supervisor opens a new `RelayStream` RPC on the same `Channel` — a new HTTP/2 stream, no new TCP connection and no new TLS handshake. The first `RelayFrame` is a `RelayInit { channel_id }` that claims the pending slot on the gateway. +4. `claim_relay` pairs the gateway-side waiter with the supervisor-side RPC via a `tokio::io::duplex(64 KiB)` pair. Subsequent `RelayFrame::data` frames carry raw SSH bytes in both directions. The supervisor is a dumb byte bridge: it has no protocol awareness of the SSH bytes flowing through. +5. Inside the sandbox pod, the supervisor connects the relay to sshd over a Unix domain socket at `/run/openshell/ssh.sock` (see `crates/openshell-driver-kubernetes/src/main.rs`). + +Security properties of this model: -- **HMAC**: `HMAC-SHA256(secret, "{token}|{timestamp}|{nonce}")`, hex-encoded. -- **Secret**: shared via `OPENSHELL_SSH_HANDSHAKE_SECRET` env var, set on both the gateway and sandbox. -- **Clock skew tolerance**: configurable via `OPENSHELL_SSH_HANDSHAKE_SKEW_SECS` (default 300 seconds). -- **Expected response**: `OK\n` from the sandbox. +- **One auth boundary.** mTLS on the `ConnectSupervisor` stream is the only identity check between gateway and sandbox. Every relay rides that same authenticated HTTP/2 connection. +- **No inbound network path into the sandbox.** The sandbox exposes no TCP port for gateway ingress; all relays are supervisor-initiated. The pod only needs egress to the gateway. +- **In-pod access control is filesystem permissions on the Unix socket.** sshd listens on `/run/openshell/ssh.sock` with the parent directory at `0700` and the socket itself at `0600`, both owned by the supervisor (root). The sandbox entrypoint runs as an unprivileged user and cannot open either. Any process in the supervisor's filesystem view that can open the socket can reach sshd — same trust model as any local Unix socket with `0600` permissions. See `crates/openshell-sandbox/src/ssh.rs:55-83`. +- **Supersede race is closed.** A supervisor reconnect registers a new `session_id` for the same sandbox id. Cleanup on the old session's task uses `remove_if_current(sandbox_id, session_id)` so a late-finishing old task cannot evict the new registration or serve relays meant for the new instance. See `SupervisorSessionRegistry::remove_if_current` in `crates/openshell-server/src/supervisor_session.rs`. +- **Pending-relay reaper.** A background task sweeps `pending_relays` entries older than 10 s (`RELAY_PENDING_TIMEOUT`). If the supervisor acknowledges `RelayOpen` but never initiates `RelayStream` — crash, deadlock, or adversarial stall — the gateway-side slot does not pin indefinitely. +- **Client-side keepalives.** The CLI's `ssh` invocation sets `ServerAliveInterval=15` / `ServerAliveCountMax=3` (`crates/openshell-cli/src/ssh.rs:150`), so a silently-dropped relay (gateway restart, supervisor restart, or adversarial TCP drop) surfaces to the user within roughly 45 s rather than hanging. -This handshake prevents direct connections to sandbox SSH ports from within the cluster, even from pods that share the network. +Observability (sandbox side, OCSF): `session_established`, `session_closed`, `session_failed`, `relay_open`, `relay_closed`, `relay_failed`, `relay_close_from_gateway` — all emitted as `NetworkActivity` events. Gateway-side OCSF emission for the same lifecycle is a tracked follow-up. ## Port Configuration @@ -325,8 +333,8 @@ graph LR CLI -- "mTLS
(cluster CA)" --> TLS SDK -- "mTLS
(cluster CA)" --> TLS TLS --> API - SBX -- "mTLS
(cluster CA)" --> TLS - API -- "SSH + NSSH1
handshake" --> SBX + SBX -- "mTLS + ConnectSupervisor
(supervisor-initiated)" --> TLS + API -- "RelayStream
(HTTP/2 on same mTLS conn)" --> SBX SBX -- "OPA policy +
process identity" --> HOSTS ``` @@ -335,8 +343,9 @@ graph LR | Boundary | Mechanism | |---|---| | External → Gateway | mTLS with cluster CA by default, or trusted reverse-proxy/Cloudflare boundary in edge mode | -| Sandbox → Gateway | mTLS with shared client cert | -| Gateway → Sandbox (SSH) | Session token + HMAC-SHA256 handshake (NSSH1) | +| Sandbox → Gateway | mTLS with shared client cert (supervisor-initiated `ConnectSupervisor` stream) | +| Gateway → Sandbox (SSH/exec) | Rides the supervisor's mTLS `ConnectSupervisor` HTTP/2 connection as a `RelayStream` — no separate gateway-to-pod connection | +| Supervisor → in-pod sshd | Unix-socket filesystem permissions (`/run/openshell/ssh.sock`, 0700 parent / 0600 socket) | | Sandbox → External (network) | OPA policy + process identity binding via `/proc` | ### What Is Not Authenticated (by Design) @@ -387,8 +396,11 @@ This section defines the primary attacker profiles, what the current design prot |---|---|---| | MITM or passive interception of gateway traffic | Mandatory mTLS with cluster CA, or trusted reverse-proxy boundary in Cloudflare mode | Default mode is direct mTLS; reverse-proxy mode shifts the outer trust boundary upstream | | Unauthenticated API/health access | mTLS by default, or Cloudflare/reverse-proxy auth in edge mode | `/health*` are direct-mTLS only in the default deployment mode | -| Forged SSH tunnel connection to sandbox | Session token validation + NSSH1 HMAC handshake | Requires token and shared handshake secret | -| Direct access to sandbox SSH port from cluster peers | NSSH1 challenge-response | Connection denied without valid signature | +| Forged SSH tunnel connection to sandbox | Session token validation at the gateway; only the supervisor's authenticated mTLS `ConnectSupervisor` stream can carry a `RelayStream` to its sandbox | Forging a relay requires stealing a valid mTLS client identity | +| Direct access to sandbox sshd from cluster peers | sshd listens on a Unix socket (`0700` parent / `0600` socket) inside the pod | No network path exists to sshd from cluster peers | +| Stale or reconnecting supervisor serves relays for a new instance | `session_id`-scoped `remove_if_current` on the registry | Old session cleanup cannot evict a newer registration | +| Supervisor acknowledges `RelayOpen` but never initiates `RelayStream` | Gateway-side pending-relay reaper (10 s timeout) | Prevents indefinite resource pinning by a buggy or malicious supervisor | +| Silent TCP drop of an in-flight relay | CLI `ServerAliveInterval=15` / `ServerAliveCountMax=3` | Client detects a dead relay within ~45 s instead of hanging | | Unauthorized outbound internet access from sandbox | OPA policy + process identity checks | Applies to sandbox egress policy layer | ### Residual Risks and Current Tradeoffs @@ -414,7 +426,7 @@ This section defines the primary attacker profiles, what the current design prot - The cluster CA is generated and distributed without interception during bootstrap. - Kubernetes secret access is restricted to intended workloads and operators. - Gateway and sandbox container images are trusted and not tampered with. -- System clocks are reasonably synchronized for timestamp-based SSH handshake checks. +- The sandbox pod's filesystem is trusted: only the supervisor process (root) can open `/run/openshell/ssh.sock`, which is enforced by the `0700` parent directory and `0600` socket permissions set at sshd start. ## Sandbox Outbound TLS (L7 Inspection) diff --git a/architecture/gateway.md b/architecture/gateway.md index 294506e57d..5dd2419af7 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -2,7 +2,9 @@ ## Overview -`openshell-server` is the gateway -- the central control plane for a cluster. It exposes two gRPC services (OpenShell and Inference) and HTTP endpoints on a single multiplexed port, manages sandbox lifecycle through Kubernetes CRDs, persists state in SQLite or Postgres, and provides SSH tunneling into sandbox pods. The gateway coordinates all interactions between clients, the Kubernetes cluster, and the persistence layer. +`openshell-server` is the gateway -- the central control plane for a cluster. It exposes two gRPC services (OpenShell and Inference) and HTTP endpoints on a single multiplexed port, manages sandbox lifecycle through a pluggable compute driver, persists state in SQLite or Postgres, and brokers SSH access into sandboxes through supervisor-initiated relay streams. The gateway coordinates all interactions between clients, the compute backend, and the persistence layer. + +Each sandbox supervisor opens a persistent inbound gRPC session (`ConnectSupervisor`); the gateway multiplexes per-invocation `RelayStream` RPCs onto the same HTTP/2 connection to move bytes between clients and the in-sandbox SSH Unix socket. The gateway does not need to know, resolve, or reach the sandbox's network address. ## Architecture Diagram @@ -11,25 +13,27 @@ The following diagram shows the major components inside the gateway process and ```mermaid graph TD Client["gRPC / HTTP Client"] + Supervisor["Sandbox Supervisor
(inbound gRPC)"] TCP["TCP Listener"] TLS["TLS Acceptor
(optional)"] - MUX["MultiplexedService"] + MUX["MultiplexedService
(HTTP/2 adaptive window)"] GRPC_ROUTER["GrpcRouter"] NAV["OpenShellServer
(OpenShell service)"] INF["InferenceServer
(Inference service)"] HTTP["HTTP Router
(Axum)"] HEALTH["Health Endpoints"] SSH_TUNNEL["SSH Tunnel
(/connect/ssh)"] + SUP_REG["SupervisorSessionRegistry"] STORE["Store
(SQLite / Postgres)"] - K8S["Kubernetes API"] - WATCHER["Sandbox Watcher"] - EVENT_TAILER["Kube Event Tailer"] + COMPUTE["ComputeRuntime"] + DRIVER["ComputeDriver
(kubernetes / vm)"] WATCH_BUS["SandboxWatchBus"] LOG_BUS["TracingLogBus"] PLAT_BUS["PlatformEventBus"] INDEX["SandboxIndex"] Client --> TCP + Supervisor --> TCP TCP --> TLS TLS --> MUX MUX -->|"content-type: application/grpc"| GRPC_ROUTER @@ -39,17 +43,16 @@ graph TD HTTP --> HEALTH HTTP --> SSH_TUNNEL NAV --> STORE - NAV --> K8S - INF --> STORE + NAV --> COMPUTE + NAV --> SUP_REG SSH_TUNNEL --> STORE - SSH_TUNNEL --> K8S - WATCHER --> K8S - WATCHER --> STORE - WATCHER --> WATCH_BUS - WATCHER --> INDEX - EVENT_TAILER --> K8S - EVENT_TAILER --> PLAT_BUS - EVENT_TAILER --> INDEX + SSH_TUNNEL --> SUP_REG + INF --> STORE + COMPUTE --> DRIVER + COMPUTE --> STORE + COMPUTE --> WATCH_BUS + COMPUTE --> INDEX + COMPUTE --> PLAT_BUS LOG_BUS --> PLAT_BUS ``` @@ -57,22 +60,23 @@ graph TD | Module | File | Purpose | |--------|------|---------| -| Entry point | `crates/openshell-server/src/main.rs` | CLI argument parsing, config assembly, tracing setup, calls `run_server` | +| Entry point | `crates/openshell-server/src/main.rs` | Thin binary wrapper that calls `cli::run_cli` | +| CLI | `crates/openshell-server/src/cli.rs` | `Args` parser, config assembly, tracing setup, calls `run_server` | | Gateway runtime | `crates/openshell-server/src/lib.rs` | `ServerState` struct, `run_server()` accept loop | -| Protocol mux | `crates/openshell-server/src/multiplex.rs` | `MultiplexService`, `MultiplexedService`, `GrpcRouter`, `BoxBody` | -| gRPC: OpenShell | `crates/openshell-server/src/grpc.rs` | `OpenShellService` -- sandbox CRUD, provider CRUD, watch, exec, SSH sessions, policy delivery | -| gRPC: Inference | `crates/openshell-server/src/inference.rs` | `InferenceService` -- cluster inference config (set/get) and sandbox inference bundle delivery | +| Protocol mux | `crates/openshell-server/src/multiplex.rs` | `MultiplexService`, `MultiplexedService`, `GrpcRouter`, `BoxBody`, HTTP/2 adaptive-window tuning | +| gRPC: OpenShell | `crates/openshell-server/src/grpc/mod.rs` | `OpenShellService` trait impl -- dispatches to per-concern handlers | +| gRPC: Sandbox/Exec | `crates/openshell-server/src/grpc/sandbox.rs` | Sandbox CRUD, `ExecSandbox`, SSH session handlers, relay-backed exec proxy | +| gRPC: Inference | `crates/openshell-server/src/inference.rs` | `InferenceService` -- cluster inference config and sandbox bundle delivery | +| Supervisor sessions | `crates/openshell-server/src/supervisor_session.rs` | `SupervisorSessionRegistry`, `handle_connect_supervisor`, `handle_relay_stream`, reaper | | HTTP | `crates/openshell-server/src/http.rs` | Health endpoints, merged with SSH tunnel router | | Browser auth | `crates/openshell-server/src/auth.rs` | Cloudflare browser login relay at `/auth/connect` | -| SSH tunnel | `crates/openshell-server/src/ssh_tunnel.rs` | HTTP CONNECT handler at `/connect/ssh` | +| SSH tunnel | `crates/openshell-server/src/ssh_tunnel.rs` | HTTP CONNECT handler at `/connect/ssh` backed by `open_relay` | | WS tunnel | `crates/openshell-server/src/ws_tunnel.rs` | WebSocket tunnel handler at `/_ws_tunnel` for Cloudflare-fronted clients | | TLS | `crates/openshell-server/src/tls.rs` | `TlsAcceptor` wrapping rustls with ALPN | | Persistence | `crates/openshell-server/src/persistence/mod.rs` | `Store` enum (SQLite/Postgres), generic object CRUD, protobuf codec | -| Persistence: SQLite | `crates/openshell-server/src/persistence/sqlite.rs` | `SqliteStore` with sqlx | -| Persistence: Postgres | `crates/openshell-server/src/persistence/postgres.rs` | `PostgresStore` with sqlx | | Compute runtime | `crates/openshell-server/src/compute/mod.rs` | `ComputeRuntime`, gateway-owned sandbox lifecycle orchestration over a compute backend | -| Compute driver: Kubernetes | `crates/openshell-driver-kubernetes/src/driver.rs` | Kubernetes CRD create/delete, endpoint resolution, watch stream, pod template translation | -| Compute driver: VM | `crates/openshell-driver-vm/src/driver.rs` | Per-sandbox microVM create/delete, localhost endpoint resolution, watch stream, supervisor-only guest boot | +| Compute driver: Kubernetes | `crates/openshell-driver-kubernetes/src/driver.rs` | Kubernetes CRD create/delete/watch, pod template translation | +| Compute driver: VM | `crates/openshell-driver-vm/src/driver.rs` | Per-sandbox microVM create/delete/watch, supervisor-only guest boot | | Sandbox index | `crates/openshell-server/src/sandbox_index.rs` | `SandboxIndex` -- in-memory name/pod-to-id correlation | | Watch bus | `crates/openshell-server/src/sandbox_watch.rs` | `SandboxWatchBus` -- in-memory broadcast for persisted sandbox updates | | Tracing bus | `crates/openshell-server/src/tracing_bus.rs` | `TracingLogBus` -- captures tracing events keyed by `sandbox_id` | @@ -81,28 +85,30 @@ Proto definitions consumed by the gateway: | Proto file | Package | Defines | |------------|---------|---------| -| `proto/openshell.proto` | `openshell.v1` | `OpenShell` service, public sandbox resource model, provider/SSH/watch messages | -| `proto/compute_driver.proto` | `openshell.compute.v1` | Internal `ComputeDriver` service, driver-native sandbox observations, endpoint resolution, compute watch stream envelopes | +| `proto/openshell.proto` | `openshell.v1` | `OpenShell` service, public sandbox resource model, provider/SSH/watch/policy messages, supervisor session messages (`ConnectSupervisor`, `RelayStream`, `RelayFrame`) | +| `proto/compute_driver.proto` | `openshell.compute.v1` | Internal `ComputeDriver` service, driver-native sandbox observations, compute watch stream envelopes | | `proto/inference.proto` | `openshell.inference.v1` | `Inference` service: `SetClusterInference`, `GetClusterInference`, `GetInferenceBundle` | | `proto/datamodel.proto` | `openshell.datamodel.v1` | `Provider` | | `proto/sandbox.proto` | `openshell.sandbox.v1` | Sandbox supervisor policy, settings, and config messages | ## Startup Sequence -The gateway boots in `main()` (`crates/openshell-server/src/main.rs`) and proceeds through these steps: +The gateway boots in `cli::run_cli` (`crates/openshell-server/src/cli.rs`) and proceeds through these steps: -1. **Install rustls crypto provider** -- `aws_lc_rs::default_provider().install_default()`. +1. **Install rustls crypto provider** -- `rustls::crypto::ring::default_provider().install_default()`. 2. **Parse CLI arguments** -- `Args::parse()` via `clap`. Every flag has a corresponding environment variable (see [Configuration](#configuration)). 3. **Initialize tracing** -- Creates a `TracingLogBus` and installs a tracing subscriber that writes to stdout and publishes log events keyed by `sandbox_id` into the bus. -4. **Build `Config`** -- Assembles a `openshell_core::Config` from the parsed arguments. +4. **Build `Config`** -- Assembles an `openshell_core::Config` from the parsed arguments. 5. **Call `run_server()`** (`crates/openshell-server/src/lib.rs`): 1. Connect to the persistence store (`Store::connect`), which auto-detects SQLite vs Postgres from the URL prefix and runs migrations. 2. Create `ComputeRuntime` with a `ComputeDriver` implementation selected by `OPENSHELL_DRIVERS`: - `kubernetes` wraps `KubernetesComputeDriver` in `ComputeDriverService`, so the gateway uses the `openshell.compute.v1.ComputeDriver` RPC surface even without transport. - `vm` spawns the standalone `openshell-driver-vm` binary as a local compute-driver process, resolves it from `--driver-dir`, conventional libexec install paths, or a sibling of the gateway binary, connects to it over a Unix domain socket, and keeps the libkrun/rootfs runtime out of the gateway binary. - 3. Build `ServerState` (shared via `Arc` across all handlers). + 3. Build `ServerState` (shared via `Arc` across all handlers), including a fresh `SupervisorSessionRegistry`. 4. **Spawn background tasks**: - - `ComputeRuntime::spawn_watchers` -- consumes the compute-driver watch stream, republishes platform events, and runs a periodic `ListSandboxes` snapshot reconcile so the store-backed public sandbox reads stay aligned with the compute driver. + - `ComputeRuntime::spawn_watchers` -- consumes the compute-driver watch stream, republishes platform events, and runs a periodic `ListSandboxes` snapshot reconcile. + - `ssh_tunnel::spawn_session_reaper` -- sweeps expired or revoked SSH session tokens from the store hourly. + - `supervisor_session::spawn_relay_reaper` -- sweeps orphaned pending relay channels every 30 seconds. 5. Create `MultiplexService`. 6. Bind `TcpListener` on `config.bind_address`. 7. Optionally create `TlsAcceptor` from cert/key files. @@ -110,7 +116,7 @@ The gateway boots in `main()` (`crates/openshell-server/src/main.rs`) and procee ## Configuration -All configuration is via CLI flags with environment variable fallbacks. The `--db-url` flag is the only required argument. +All configuration is via CLI flags with environment variable fallbacks. The `--db-url` and `--ssh-handshake-secret` flags are required. | Flag | Env Var | Default | Description | |------|---------|---------|-------------| @@ -125,7 +131,7 @@ All configuration is via CLI flags with environment variable fallbacks. The `--d | `--db-url` | `OPENSHELL_DB_URL` | *required* | Database URL (`sqlite:...` or `postgres://...`). The Helm chart defaults to `sqlite:/var/openshell/openshell.db` (persistent volume). In-memory SQLite (`sqlite::memory:?cache=shared`) works for ephemeral/test environments but data is lost on restart. | | `--sandbox-namespace` | `OPENSHELL_SANDBOX_NAMESPACE` | `default` | Kubernetes namespace for sandbox CRDs | | `--sandbox-image` | `OPENSHELL_SANDBOX_IMAGE` | None | Default container image for sandbox pods | -| `--grpc-endpoint` | `OPENSHELL_GRPC_ENDPOINT` | None | gRPC endpoint reachable from within the cluster (for sandbox callbacks) | +| `--grpc-endpoint` | `OPENSHELL_GRPC_ENDPOINT` | None | gRPC endpoint reachable from within the cluster (for supervisor callbacks) | | `--drivers` | `OPENSHELL_DRIVERS` | `kubernetes` | Compute backend to use. Current options are `kubernetes` and `vm`. | | `--vm-driver-state-dir` | `OPENSHELL_VM_DRIVER_STATE_DIR` | `target/openshell-vm-driver` | Host directory for VM sandbox rootfs, console logs, and runtime state | | `--driver-dir` | `OPENSHELL_DRIVER_DIR` | unset | Override directory for `openshell-driver-vm`. When unset, the gateway searches `~/.local/libexec/openshell`, `/usr/local/libexec/openshell`, `/usr/local/libexec`, then a sibling binary. | @@ -138,9 +144,8 @@ All configuration is via CLI flags with environment variable fallbacks. The `--d | `--ssh-gateway-host` | `OPENSHELL_SSH_GATEWAY_HOST` | `127.0.0.1` | Public hostname returned in SSH session responses | | `--ssh-gateway-port` | `OPENSHELL_SSH_GATEWAY_PORT` | `8080` | Public port returned in SSH session responses | | `--ssh-connect-path` | `OPENSHELL_SSH_CONNECT_PATH` | `/connect/ssh` | HTTP path for SSH CONNECT/upgrade | -| `--sandbox-ssh-port` | `OPENSHELL_SANDBOX_SSH_PORT` | `2222` | SSH listen port inside sandbox pods | -| `--ssh-handshake-secret` | `OPENSHELL_SSH_HANDSHAKE_SECRET` | None | Shared HMAC-SHA256 secret for gateway-to-sandbox handshake | -| `--ssh-handshake-skew-secs` | `OPENSHELL_SSH_HANDSHAKE_SKEW_SECS` | `300` | Allowed clock skew (seconds) for SSH handshake timestamps | + +The sandbox-side SSH listener is a Unix domain socket inside the sandbox. The path defaults to `/run/openshell/ssh.sock` and is configured on the compute driver (e.g. `openshell-driver-kubernetes --sandbox-ssh-socket-path`). The gateway never dials this socket itself; the supervisor bridges it onto a `RelayStream` when asked. ## Shared State @@ -157,15 +162,17 @@ pub struct ServerState { pub ssh_connections_by_token: Mutex>, pub ssh_connections_by_sandbox: Mutex>, pub settings_mutex: tokio::sync::Mutex<()>, + pub supervisor_sessions: SupervisorSessionRegistry, } ``` - **`store`** -- persistence backend (SQLite or Postgres) for all object types. -- **`compute`** -- gateway-owned compute orchestration. Persists sandbox lifecycle transitions, validates create requests through the compute backend, resolves exec/SSH endpoints, consumes the backend watch stream, and periodically reconciles the store against `ComputeDriver/ListSandboxes` snapshots. +- **`compute`** -- gateway-owned compute orchestration. Persists sandbox lifecycle transitions, validates create requests through the compute backend, consumes the backend watch stream, and periodically reconciles the store against `ComputeDriver/ListSandboxes` snapshots. - **`sandbox_index`** -- in-memory bidirectional index mapping sandbox names and agent pod names to sandbox IDs. Updated from compute-driver sandbox snapshots. - **`sandbox_watch_bus`** -- `broadcast`-based notification bus keyed by sandbox ID. Producers call `notify(&id)` when the persisted sandbox record changes; consumers in `WatchSandbox` streams receive `()` signals and re-read the record. - **`tracing_log_bus`** -- captures `tracing` events that include a `sandbox_id` field and republishes them as `SandboxLogLine` messages. Maintains a per-sandbox tail buffer (default 200 entries). Also contains a nested `PlatformEventBus` for compute-driver platform events. -- **`settings_mutex`** -- serializes settings mutations (global and sandbox) to prevent read-modify-write races. Held for the duration of any setting set/delete or global policy set/delete operation. See [Gateway Settings Channel](gateway-settings.md#global-policy-lifecycle). +- **`supervisor_sessions`** -- tracks the live `ConnectSupervisor` session per sandbox and the set of pending relay channels awaiting the supervisor's `RelayStream` dial-back. See [Supervisor Sessions](#supervisor-sessions). +- **`settings_mutex`** -- serializes settings mutations (global and sandbox) to prevent read-modify-write races. See [Gateway Settings Channel](gateway-settings.md#global-policy-lifecycle). ## Protocol Multiplexing @@ -176,8 +183,9 @@ All traffic (gRPC and HTTP) shares a single TCP port. Multiplexing happens at th `MultiplexService::serve()` (`crates/openshell-server/src/multiplex.rs`) creates per-connection service instances: 1. Each accepted TCP stream (optionally TLS-wrapped) is passed to `hyper_util::server::conn::auto::Builder`, which auto-negotiates HTTP/1.1 or HTTP/2. -2. The builder calls `serve_connection_with_upgrades()`, which supports HTTP upgrades (needed for the SSH tunnel's CONNECT method). -3. For each request, `MultiplexedService` inspects the `content-type` header: +2. The HTTP/2 side is built with `adaptive_window(true)`. Hyper/h2 auto-sizes the per-stream flow-control window based on measured bandwidth-delay product, so bulk byte transfers on `RelayStream` (and `ExecSandbox` / `PushSandboxLogs`) are not throttled by the default 64 KiB window. Idle streams stay cheap; active streams grow as needed. +3. The builder calls `serve_connection_with_upgrades()`, which supports HTTP upgrades (needed for the SSH tunnel's CONNECT method). +4. For each request, `MultiplexedService` inspects the `content-type` header: - **Starts with `application/grpc`** -- routes to `GrpcRouter`. - **Anything else** -- routes to the Axum HTTP router. @@ -202,32 +210,147 @@ When TLS is enabled (`crates/openshell-server/src/tls.rs`): - Supports PKCS#1, PKCS#8, and SEC1 private key formats. - The TLS handshake happens before the stream reaches Hyper's auto builder, so ALPN negotiation and HTTP version detection work together transparently. - Certificates are generated at cluster bootstrap time by the `openshell-bootstrap` crate using `rcgen`, not by a Helm Job. The bootstrap reconciles three K8s secrets: `openshell-server-tls` (server cert+key), `openshell-server-client-ca` (CA cert), and `openshell-client-tls` (client cert+key+CA, shared by CLI and sandbox pods). -- **Certificate lifetime**: Certificates use `rcgen` defaults (effectively never expire), which is appropriate for an internal dev-cluster PKI where certs are ephemeral to the cluster's lifetime. -- **Redeploy behavior**: On redeploy, existing cluster TLS secrets are loaded and reused if they are complete and valid PEM. If secrets are missing, incomplete, or malformed, fresh PKI is generated. If rotation occurs and the openshell workload is already running, the bootstrap performs a rollout restart and waits for completion before persisting CLI-side credentials. +- Sandbox supervisors reuse the shared client cert to authenticate their `ConnectSupervisor` and `RelayStream` calls over the same mTLS channel. + +## Supervisor Sessions + +The gateway brokers all byte-level access into a sandbox through a two-plane design on a single HTTP/2 connection initiated by the supervisor: + +1. **Control plane** -- `ConnectSupervisor(stream SupervisorMessage) returns (stream GatewayMessage)`. Long-lived, one per sandbox. Carries `SupervisorHello`, `SessionAccepted`/`SessionRejected`, heartbeats, and `RelayOpen`/`RelayClose` control messages. +2. **Data plane** -- `RelayStream(stream RelayFrame) returns (stream RelayFrame)`. One short-lived call per SSH or exec invocation. The first inbound frame is a `RelayInit { channel_id }`; subsequent frames carry raw bytes in `RelayFrame.data` in either direction. + +Both RPCs are defined in `proto/openshell.proto` and ride the same TCP + TLS + HTTP/2 connection from the supervisor. No new TLS handshake, no reverse HTTP CONNECT, no direct gateway-to-pod dial. + +### `SupervisorSessionRegistry` + +`crates/openshell-server/src/supervisor_session.rs` defines `SupervisorSessionRegistry`, a single instance of which lives on `ServerState.supervisor_sessions`. It holds two maps guarded by `std::sync::Mutex`: + +- `sessions: HashMap` -- one entry per connected supervisor. Each `LiveSession` carries a unique `session_id`, the `mpsc::Sender` for the outbound stream, and a connection timestamp. +- `pending_relays: HashMap` -- one entry per in-flight `open_relay` call awaiting the supervisor's `RelayStream` dial-back. Each `PendingRelay` wraps a `oneshot::Sender` and a creation timestamp. + +Core operations: + +| Method | Purpose | +|--------|---------| +| `register(sandbox_id, session_id, tx)` | Insert a live session; returns the previous session's sender (if any) so the caller can close it. Used by `handle_connect_supervisor` when a supervisor reconnects. | +| `remove_if_current(sandbox_id, session_id)` | Remove the session only if its `session_id` still matches. Guards against the supersede race where an old session's cleanup task fires after a newer session already registered. | +| `open_relay(sandbox_id, session_wait_timeout)` | Wait up to `session_wait_timeout` for a live session, allocate a fresh `channel_id` (UUID v4), insert the pending slot, send `RelayOpen { channel_id }` to the supervisor, and return `(channel_id, oneshot::Receiver)`. The receiver resolves once the supervisor's `RelayStream` arrives and `claim_relay` pairs them up. | +| `claim_relay(channel_id)` | Consume the pending slot, construct a `tokio::io::duplex(64 KiB)` pair, hand the gateway-side half to the waiter via the oneshot, and return the supervisor-side half to `handle_relay_stream`. | +| `reap_expired_relays()` | Drop pending relays older than 10 s. Called by `spawn_relay_reaper` on a 30 s cadence. | + +Session wait uses exponential backoff from 100 ms to 2 s while polling the sessions map. Pending-relay expiry is fixed at `RELAY_PENDING_TIMEOUT = 10 s`. + +### `handle_connect_supervisor` + +Lifecycle of a supervisor session: + +1. Read the first `SupervisorMessage`; require `payload = Hello { sandbox_id, instance_id }` and a non-empty `sandbox_id`. +2. Allocate a fresh `session_id` (UUID v4) and create an `mpsc::channel::(64)` for the outbound stream. +3. Call `registry.register(...)`. If it returns a previous sender, log that the previous session was superseded (dropping the previous `tx` closes the old outbound stream). +4. Send `SessionAccepted { session_id, heartbeat_interval_secs: 15 }`. If the send fails, call `remove_if_current` (so a concurrent reconnect isn't evicted) and return `Internal`. +5. Spawn a session loop that `select!`s between inbound messages and a 15 s heartbeat timer. Inbound heartbeats are silent; `RelayOpenResult` is logged; `RelayClose` is logged; unknown payloads are logged as warnings. +6. When the loop exits (inbound EOF, inbound error, or outbound channel closed), `remove_if_current` drops the registration -- unless a newer session has already replaced it. + +### `handle_relay_stream` + +Lifecycle of one relay call: + +1. Read the first inbound `RelayFrame`; require `payload = Init { channel_id }` with a non-empty `channel_id`. Reject anything else with `InvalidArgument`. +2. Call `registry.claim_relay(channel_id)`. Returns `NotFound` if the channel is unknown or already expired, `DeadlineExceeded` if older than 10 s, or `Internal` if the waiter has dropped the oneshot receiver. +3. Split the supervisor-side `DuplexStream` into read and write halves and spawn two tasks: + - **Supervisor → gateway**: pull `RelayFrame`s from the inbound stream, accept `Data(bytes)`, write to the duplex write-half. On non-data frames, warn and break. Best-effort `shutdown()` on exit so the reader sees EOF. + - **Gateway → supervisor**: read up to `RELAY_STREAM_CHUNK_SIZE = 16 KiB` at a time from the duplex read-half and emit `RelayFrame { Data }` messages on an outbound `mpsc::channel(16)`. +4. Return the outbound receiver as the RPC response stream. + +### Connect Flow (SSH Tunnel) + +```mermaid +sequenceDiagram + participant Client as SSH client + participant GW as Gateway
(/connect/ssh) + participant Reg as SupervisorSessionRegistry + participant Sup as Sandbox Supervisor + participant Daemon as In-sandbox sshd
(Unix socket) + + Client->>GW: CONNECT /connect/ssh
x-sandbox-id, x-sandbox-token + GW->>GW: validate session + sandbox Ready + GW->>Reg: open_relay(sandbox_id, 30s) + Reg->>Sup: GatewayMessage::RelayOpen { channel_id } + Note over Reg: waits for RelayStream on channel_id + Sup->>Daemon: connect to Unix socket + Sup->>GW: RelayStream(RelayFrame::Init { channel_id }) + GW->>Reg: claim_relay(channel_id) + Reg-->>Sup: supervisor-side DuplexStream + Reg-->>GW: gateway-side DuplexStream + GW-->>Client: 200 OK + HTTP upgrade + Client<<->>GW: copy_bidirectional(upgraded, duplex) + GW<<->>Sup: RelayFrame::Data in both directions + Sup<<->>Daemon: raw SSH bytes +``` + +Timeouts on the tunnel path: + +- `open_relay` session wait: **30 s**. A first `sandbox connect` immediately after `sandbox create` must cover the supervisor's initial TLS + gRPC handshake on a cold pod. +- `relay_rx` delivery timeout: 10 s. Covers the round-trip from the `RelayOpen` message to the supervisor's `RelayStream` dial-back. + +Per-token and per-sandbox concurrent-tunnel limits (3 and 20 respectively) are still enforced before the upgrade. + +### Exec Flow + +`ExecSandbox` reuses the same machinery from `grpc/sandbox.rs`: + +1. Validate the request (`sandbox_id`, `command`, env-key format, other field rules), fetch the sandbox, require `Ready` phase. +2. `state.supervisor_sessions.open_relay(&sandbox.id, 15s)` -- shorter timeout than SSH connect, because exec is typically called mid-lifetime after the supervisor session is already established. +3. Wait up to 10 s for the relay `DuplexStream`. +4. `stream_exec_over_relay`: bind an ephemeral localhost TCP listener, bridge that single-use TCP socket to the relay duplex, and drive a `russh` client through the local port. The `russh` session opens a channel, executes the shell-escaped command, and streams `ExecSandboxStdout`/`ExecSandboxStderr` chunks to the caller. On completion, send `ExecSandboxExit { exit_code }`. +5. On timeout (if `timeout_seconds > 0`), emit exit code 124 (matching `timeout(1)`). + +The supervisor-side SSH daemon is an SSH server bound to a Unix domain socket inside the sandbox's filesystem. Filesystem permissions on that socket are the only access-control boundary between the supervisor bridge and the daemon; all higher-level authorization is enforced at `CreateSshSession` / `ExecSandbox` in the gateway. + +### Regression Coverage + +`crates/openshell-server/tests/supervisor_relay_integration.rs` is the regression guard for the `RelayStream` wire protocol. It stands up an in-process tonic server that mounts the real `handle_relay_stream` behind `MultiplexedService`, connects a mock supervisor client over a real tonic `Channel`, and exercises the registry's `open_relay` → `claim_relay` pairing end to end with `tokio::io::duplex` bridging. The five test cases cover: + +- Round-trip bytes from gateway to supervisor and back (echo loop). +- Clean close when the gateway drops the relay. +- EOF propagation when the supervisor closes its outbound sender. +- `Unavailable` when `open_relay` is called without a registered session. +- Concurrent `RelayStream` calls multiplexed independently on the same connection. + +These complement the unit tests inside `supervisor_session.rs` (registry-only behavior) and the live cluster tests (full CLI → gateway → sandbox path). ## gRPC Services ### OpenShell Service -Defined in `proto/openshell.proto`, implemented in `crates/openshell-server/src/grpc.rs` as `OpenShellService`. +Defined in `proto/openshell.proto`, implemented in `crates/openshell-server/src/grpc/mod.rs` as `OpenShellService`. Per-concern handlers live in `crates/openshell-server/src/grpc/` submodules. #### Sandbox Management | RPC | Description | Key behavior | |-----|-------------|--------------| | `Health` | Returns service status and version | Always returns `HEALTHY` with `CARGO_PKG_VERSION` | -| `CreateSandbox` | Create a new sandbox | Validates spec and policy, validates provider names exist (fail-fast), persists to store, creates Kubernetes CRD. On K8s 409 conflict or error, rolls back the store record and index entry. | +| `CreateSandbox` | Create a new sandbox | Validates spec and policy, validates provider names exist (fail-fast), persists to store, creates the compute-driver sandbox. On driver failure, rolls back the store record and index entry. | | `GetSandbox` | Fetch sandbox by name | Looks up by name via `store.get_message_by_name()` | | `ListSandboxes` | List sandboxes | Paginated (default limit 100), decodes protobuf payloads from store records | -| `DeleteSandbox` | Delete sandbox by name | Sets phase to `Deleting`, persists, notifies watch bus, then deletes the Kubernetes CRD. Cleans up store if the CRD was already gone. | +| `DeleteSandbox` | Delete sandbox by name | Sets phase to `Deleting`, persists, notifies watch bus, then deletes via the compute driver. Cleans up store if the sandbox was already gone. | | `WatchSandbox` | Stream sandbox updates | Server-streaming RPC. See [Watch Sandbox Stream](#watch-sandbox-stream) below. | -| `ExecSandbox` | Execute command in sandbox | Server-streaming RPC. See [Remote Exec via SSH](#remote-exec-via-ssh) below. | +| `ExecSandbox` | Execute command in sandbox | Server-streaming RPC; data plane runs through `SupervisorSessionRegistry::open_relay`. See [Exec Flow](#exec-flow). | + +#### Supervisor Session + +| RPC | Description | +|-----|-------------| +| `ConnectSupervisor` | Persistent bidi stream from the sandbox supervisor. Carries hello/accept/heartbeat/`RelayOpen`/`RelayClose`. One session per sandbox; reconnects supersede. | +| `RelayStream` | Per-invocation bidi byte bridge. Supervisor initiates after receiving `RelayOpen`; first frame is `RelayInit { channel_id }`; subsequent frames carry raw bytes. | + +Neither RPC is called by end users. They are the private control/data plane between the gateway and each sandbox supervisor. #### SSH Session Management | RPC | Description | |-----|-------------| -| `CreateSshSession` | Creates a session token for a `Ready` sandbox. Persists an `SshSession` record and returns gateway connection details (host, port, scheme, connect path). | +| `CreateSshSession` | Creates a session token for a `Ready` sandbox. Persists an `SshSession` record and returns gateway connection details (host, port, scheme, connect path). The resulting token is presented on the `/connect/ssh` HTTP CONNECT request. | | `RevokeSshSession` | Marks a session as revoked by setting `session.revoked = true` in the store. | #### Provider Management @@ -244,17 +367,17 @@ Full CRUD for `Provider` objects, which store typed credentials (e.g., API keys #### Policy, Settings, and Provider Environment Delivery -These RPCs are called by sandbox pods at startup and during runtime polling. +These RPCs are called by sandbox supervisors at startup and during runtime polling. | RPC | Description | |-----|-------------| -| `GetSandboxSettings` | Returns effective sandbox config looked up by sandbox ID: policy payload, policy metadata (version, hash, source, `global_policy_version`), merged effective settings, and a `config_revision` fingerprint for change detection. Two-tier resolution: registered keys start unset, sandbox values overlay, global values override. The reserved `policy` key in global settings can override the sandbox's own policy. When a global policy is active, `policy_source` is `GLOBAL` and `global_policy_version` carries the active revision number. See [Gateway Settings Channel](gateway-settings.md). | -| `GetGatewaySettings` | Returns gateway-global settings only (excluding the reserved `policy` key). Returns registered keys with empty values when unconfigured, and a monotonic `settings_revision`. | +| `GetSandboxConfig` | Returns effective sandbox config looked up by sandbox ID: policy payload, policy metadata (version, hash, source, `global_policy_version`), merged effective settings, and a `config_revision` fingerprint for change detection. Two-tier resolution: registered keys start unset, sandbox values overlay, global values override. The reserved `policy` key in global settings can override the sandbox's own policy. When a global policy is active, `policy_source` is `GLOBAL` and `global_policy_version` carries the active revision number. See [Gateway Settings Channel](gateway-settings.md). | +| `GetGatewayConfig` | Returns gateway-global settings only (excluding the reserved `policy` key). Returns registered keys with empty values when unconfigured, and a monotonic `settings_revision`. | | `GetSandboxProviderEnvironment` | Resolves provider credentials into environment variables for a sandbox. Iterates the sandbox's `spec.providers` list, fetches each `Provider`, and collects credential key-value pairs. First provider wins on duplicate keys. Skips credential keys that do not match `^[A-Za-z_][A-Za-z0-9_]*$`. | #### Policy Recommendation (Network Rules) -These RPCs support the sandbox-initiated policy recommendation pipeline. The sandbox generates proposals via its mechanistic mapper and submits them; the gateway validates, persists, and manages the approval workflow. See [architecture/policy-advisor.md](policy-advisor.md) for the full pipeline design. +These RPCs support the sandbox-initiated policy recommendation pipeline. The sandbox generates proposals via its mechanistic mapper and submits them; the gateway validates, persists, and manages the approval workflow. See [policy-advisor.md](policy-advisor.md) for the full pipeline design. | RPC | Description | |-----|-------------| @@ -327,9 +450,9 @@ The HTTP router (`crates/openshell-server/src/http.rs`) merges two sub-routers: | Path | Method | Response | |------|--------|----------| -| `/connect/ssh` | CONNECT | Upgrades the connection to a bidirectional TCP tunnel to a sandbox pod's SSH port | +| `/connect/ssh` | CONNECT | Upgrades the connection to a bidirectional byte bridge tunneled through `SupervisorSessionRegistry::open_relay` | -See [SSH Tunnel Gateway](#ssh-tunnel-gateway) for details. +See [Connect Flow (SSH Tunnel)](#connect-flow-ssh-tunnel) for details. ### Cloudflare Endpoints @@ -340,7 +463,7 @@ See [SSH Tunnel Gateway](#ssh-tunnel-gateway) for details. ## Watch Sandbox Stream -The `WatchSandbox` RPC (`crates/openshell-server/src/grpc.rs`) provides a multiplexed server-streaming response that can include sandbox status snapshots, gateway log lines, and platform events. +The `WatchSandbox` RPC (`crates/openshell-server/src/grpc/`) provides a multiplexed server-streaming response that can include sandbox status snapshots, gateway log lines, and platform events. ### Request Options @@ -348,7 +471,7 @@ The `WatchSandboxRequest` controls what the stream includes: - `follow_status` -- subscribe to `SandboxWatchBus` notifications and re-read the sandbox record on each change. - `follow_logs` -- subscribe to `TracingLogBus` for gateway log lines correlated by `sandbox_id`. -- `follow_events` -- subscribe to `PlatformEventBus` for Kubernetes events correlated to the sandbox. +- `follow_events` -- subscribe to `PlatformEventBus` for compute-driver platform events correlated to the sandbox. - `log_tail_lines` -- replay the last N log lines before following (default 200). - `stop_on_terminal` -- end the stream when the sandbox reaches the `Ready` phase. Note: `Error` phase does not stop the stream because it may be transient (e.g., `ReconcilerError`). @@ -367,8 +490,8 @@ The `WatchSandboxRequest` controls what the stream includes: ```mermaid graph LR - SW["spawn_sandbox_watcher"] - ET["spawn_kube_event_tailer"] + CW["ComputeRuntime watcher"] + PE["Platform events
(driver watch)"] TL["SandboxLogLayer
(tracing layer)"] WB["SandboxWatchBus
(broadcast per ID)"] @@ -377,9 +500,9 @@ graph LR WS["WatchSandbox stream"] - SW -->|"notify(id)"| WB + CW -->|"notify(id)"| WB TL -->|"publish(id, log_event)"| LB - ET -->|"publish(id, platform_event)"| PB + PE -->|"publish(id, platform_event)"| PB WB -->|"subscribe(id)"| WS LB -->|"subscribe(id)"| WS @@ -397,52 +520,6 @@ Broadcast lag is translated to `Status::resource_exhausted` via `broadcast_to_st **Validation:** `WatchSandbox` validates that the sandbox exists before subscribing to any bus, preventing entries from being created for non-existent IDs. `PushSandboxLogs` validates sandbox existence once on the first batch of the stream. -## Remote Exec via SSH - -The `ExecSandbox` RPC (`crates/openshell-server/src/grpc.rs`) executes a command inside a sandbox pod over SSH and streams stdout/stderr/exit back to the client. - -### Execution Flow - -1. Validate request: `sandbox_id`, `command`, and environment key format (`^[A-Za-z_][A-Za-z0-9_]*$`). -2. Verify sandbox exists and is in `Ready` phase. -3. Resolve target: prefer agent pod IP, fall back to Kubernetes service DNS (`..svc.cluster.local`). If the sandbox is not connectable yet (for example the pod exists but has no IP), the gateway returns `FAILED_PRECONDITION` instead of surfacing the condition as an internal server fault. -4. Build the remote command string: sort environment variables, shell-escape all values, prepend `cd &&` if `workdir` is set. -5. **Start a single-use SSH proxy**: binds an ephemeral local TCP port, accepts one connection, performs the NSSH1 handshake with the sandbox, and bidirectionally copies data. -6. **Connect via `russh`**: establishes an SSH connection through the local proxy, authenticates with `none` auth as user `sandbox`, opens a session channel, and executes the command. -7. Stream `ExecSandboxStdout`, `ExecSandboxStderr` chunks as they arrive, then send `ExecSandboxExit` with the exit code. -8. On timeout (if `timeout_seconds > 0`), send exit code 124 (matching the `timeout(1)` convention). - -### NSSH1 Handshake Protocol - -The single-use SSH proxy and the SSH tunnel endpoint both use the same handshake: - -``` -NSSH1 \n -``` - -- `token` -- session token or a one-time UUID. -- `timestamp` -- Unix epoch seconds. -- `nonce` -- UUID v4. -- `hmac_signature` -- `HMAC-SHA256(secret, "{token}|{timestamp}|{nonce}")`, hex-encoded. -- Expected response: `OK\n` from the sandbox. - -The `ssh_handshake_skew_secs` configuration controls how much clock skew is tolerated. - -## SSH Tunnel Gateway - -The SSH tunnel endpoint (`crates/openshell-server/src/ssh_tunnel.rs`) allows external SSH clients to reach sandbox pods through the gateway using HTTP CONNECT upgrades. - -### Request Flow - -1. Client sends `CONNECT /connect/ssh` with headers `x-sandbox-id` and `x-sandbox-token`. -2. Handler validates the method is CONNECT, extracts headers. -3. Fetches the `SshSession` from the store by token; rejects if revoked or if `sandbox_id` does not match. -4. Fetches the `Sandbox`; rejects if not in `Ready` phase. -5. Resolves the connect target: agent pod IP if available, otherwise Kubernetes service DNS. -6. Returns `200 OK`, then upgrades the connection via `hyper::upgrade::on()`. -7. In a spawned task: connects to the sandbox's SSH port, performs the NSSH1 handshake, then bidirectionally copies bytes between the upgraded HTTP connection and the sandbox TCP stream. -8. On completion, gracefully shuts down the write-half of the upgraded connection for clean EOF handling. - ## Persistence Layer ### Store Architecture @@ -508,28 +585,33 @@ The Helm chart template is at `deploy/helm/openshell/templates/statefulset.yaml` - **Get / Delete**: Operate by primary key (`id`), filtered by `object_type`. - **List**: Pages by `limit` + `offset` with deterministic ordering: `ORDER BY created_at_ms ASC, name ASC`. The secondary sort on `name` prevents unstable ordering when rows share the same millisecond timestamp. -## Kubernetes Integration +## Compute Driver Integration -### Sandbox CRD Management +### Kubernetes Driver `KubernetesComputeDriver` (`crates/openshell-driver-kubernetes/src/driver.rs`) manages `agents.x-k8s.io/v1alpha1/Sandbox` CRDs behind the gateway's compute interface. The gateway binds to that driver through `ComputeDriverService` (`crates/openshell-driver-kubernetes/src/grpc.rs`) in-process, so the same `openshell.compute.v1.ComputeDriver` request and response types are exercised whether the driver is invoked locally or served over gRPC. - **Get**: `GetSandbox` looks up a sandbox CRD by name and returns a driver-native platform observation (`openshell.compute.v1.DriverSandbox`) with raw status and condition data from the object. - **List**: `ListSandboxes` enumerates sandbox CRDs and returns driver-native platform observations for each, sorted by name for stable results. -- **Create**: Translates an internal `openshell.compute.v1.DriverSandbox` message into a Kubernetes `DynamicObject` with labels (`openshell.ai/sandbox-id`, `openshell.ai/managed-by: openshell`) and a spec that includes the pod template, environment variables, and gateway-required env vars (`OPENSHELL_SANDBOX_ID`, `OPENSHELL_ENDPOINT`, `OPENSHELL_SSH_LISTEN_ADDR`, etc.). When callers do not provide custom `volumeClaimTemplates`, the driver injects a default `workspace` PVC and mounts it at `/sandbox` so the default sandbox home/workdir survives pod rescheduling. +- **Create**: Translates an internal `openshell.compute.v1.DriverSandbox` message into a Kubernetes `DynamicObject` with labels (`openshell.ai/sandbox-id`, `openshell.ai/managed-by: openshell`) and a spec that includes the pod template, environment variables, and gateway-required env vars (`OPENSHELL_SANDBOX_ID`, `OPENSHELL_ENDPOINT`, `OPENSHELL_SSH_SOCKET_PATH`, etc.). `OPENSHELL_SSH_SOCKET_PATH` is set from the driver's `--sandbox-ssh-socket-path` flag (default `/run/openshell/ssh.sock`) so the in-sandbox SSH daemon binds a Unix socket rather than a TCP port. When callers do not provide custom `volumeClaimTemplates`, the driver injects a default `workspace` PVC and mounts it at `/sandbox` so the default sandbox home/workdir survives pod rescheduling. - **Delete**: Calls the Kubernetes API to delete the CRD by name. Returns `false` if already gone (404). -- **Stop**: `proto/compute_driver.proto` now reserves `StopSandbox` for a non-destructive lifecycle transition. Resume is intentionally not a dedicated compute-driver RPC; the gateway is expected to auto-resume a stopped sandbox when a client connects or executes into it. -- **Pod IP resolution**: `agent_pod_ip()` fetches the agent pod and reads `status.podIP`. +- **Stop**: `proto/compute_driver.proto` reserves `StopSandbox` for a non-destructive lifecycle transition. Resume is intentionally not a dedicated compute-driver RPC; the gateway auto-resumes a stopped sandbox when a client connects or executes into it. -### Sandbox Watcher +The gateway reaches the sandbox exclusively through the supervisor-initiated `ConnectSupervisor` session, so the driver never returns sandbox network endpoints. -The Kubernetes driver emits `WatchSandboxes` events through `proto/compute_driver.proto`. `ComputeRuntime` consumes that stream, translates the driver-native snapshots into public `openshell.v1.Sandbox` resources, derives the public phase, and applies the results to the store. +### VM Driver + +`VmDriver` (`crates/openshell-driver-vm/src/driver.rs`) is served by the standalone `openshell-driver-vm` process. The gateway spawns that binary on demand and talks to it over the internal `openshell.compute.v1.ComputeDriver` gRPC contract via a Unix domain socket. + +- **Create**: The VM driver process allocates a sandbox-specific rootfs from its own embedded `rootfs.tar.zst`, injects an explicitly configured guest mTLS bundle when the gateway callback endpoint is `https://`, then re-execs itself in a hidden helper mode that loads libkrun directly and boots the supervisor. +- **Networking**: The helper starts an embedded `gvproxy`, wires it into libkrun as virtio-net, and gives the guest outbound connectivity. No inbound TCP listener is needed — the supervisor reaches the gateway over its outbound `ConnectSupervisor` stream. +- **Gateway callback**: The guest init script configures `eth0` for gvproxy networking, prefers the configured `OPENSHELL_GRPC_ENDPOINT`, and falls back to host aliases or the gvproxy gateway IP (`192.168.127.1`) when local hostname resolution is unavailable on macOS. +- **Guest boot**: The sandbox guest runs a minimal init script that starts `openshell-sandbox` directly as PID 1 inside the VM. +- **Watch stream**: Emits provisioning, ready, error, deleting, deleted, and platform-event updates so the gateway store remains the durable source of truth. -- **Applied**: Extracts the sandbox ID from labels (or falls back to name prefix stripping), reads the CRD status, emits a driver-native snapshot, and lets the gateway translate that into the stored public sandbox record. Notifies the watch bus. -- **Deleted**: Removes the sandbox record from the store and the index. Notifies the watch bus. -- **Restarted**: Re-processes all objects (full resync). +### Compute Runtime -In addition to the watch stream, `ComputeRuntime` periodically calls `ComputeDriver/ListSandboxes` through the in-process `ComputeDriverService` and reconciles the store to that full driver snapshot. Public `GetSandbox` and `ListSandboxes` handlers remain store-backed, but the store is refreshed from the driver on a timer so the gateway still exercises the compute-driver RPC surface for reconciliation. +`ComputeRuntime` consumes the driver-native watch stream from `WatchSandboxes`, translates the snapshots into public `openshell.v1.Sandbox` resources, derives the public phase, and applies the results to the store. In parallel, it periodically calls `ListSandboxes` and reconciles the store to the full driver snapshot; public `GetSandbox` and `ListSandboxes` handlers remain store-backed but are refreshed from the driver on a timer. ### Gateway Phase Derivation @@ -546,7 +628,7 @@ In addition to the watch stream, `ComputeRuntime` periodically calls `ComputeDri **Transient reasons** (will retry, stay in `Provisioning`): `ReconcilerError`, `DependenciesNotReady`. All other `Ready=False` reasons are treated as terminal failures (`Error` phase). -### Kubernetes Event Tailer +### Kubernetes Event Correlation The Kubernetes driver also watches namespace-scoped Kubernetes `Event` objects and correlates them to sandbox IDs before emitting them as compute-driver platform events: @@ -556,17 +638,6 @@ The Kubernetes driver also watches namespace-scoped Kubernetes `Event` objects a Matched events are published to the `PlatformEventBus` as `SandboxStreamEvent::Event` payloads. -## VM Driver - -`VmDriver` (`crates/openshell-driver-vm/src/driver.rs`) is served by the standalone `openshell-driver-vm` process. The gateway spawns that binary on demand, talks to it over the internal `openshell.compute.v1.ComputeDriver` gRPC contract via a Unix domain socket, and keeps VM runtime dependencies out of `openshell-server`. - -- **Create**: The VM driver process allocates a localhost SSH port, prepares a sandbox-specific rootfs from its own embedded `rootfs.tar.zst`, injects an explicitly configured guest mTLS bundle when the gateway callback endpoint is `https://`, then re-execs itself in a hidden helper mode that loads libkrun directly and boots `/srv/openshell-vm-sandbox-init.sh`. -- **Networking**: The helper starts an embedded `gvproxy`, wires it into libkrun as virtio-net, and exposes the single inbound SSH port (`host_port:2222`) through gvproxy’s forwarder API. This keeps VM launch inside `openshell-driver-vm` without depending on the `openshell-vm` binary. -- **Gateway callback**: The guest init script configures `eth0` for gvproxy networking, prefers the configured `OPENSHELL_GRPC_ENDPOINT`, and falls back to host aliases or the gvproxy gateway IP (`192.168.127.1`) when local hostname resolution is unavailable on macOS. -- **Guest boot**: The sandbox guest runs a minimal init script that skips k3s and starts `openshell-sandbox` directly as PID 1 inside the VM. -- **Endpoint resolution**: Returns `127.0.0.1:` for SSH/exec transport. -- **Watch stream**: Emits provisioning, ready, error, deleting, deleted, and platform-event updates so the gateway store remains the durable source of truth. - ## Sandbox Index `SandboxIndex` (`crates/openshell-server/src/sandbox_index.rs`) maintains two in-memory maps protected by an `RwLock`: @@ -574,29 +645,35 @@ Matched events are published to the `PlatformEventBus` as `SandboxStreamEvent::E - `sandbox_name_to_id: HashMap` - `agent_pod_to_id: HashMap` -Updated by the sandbox watcher on every Applied event and by gRPC handlers during sandbox creation. Used by the event tailer to map Kubernetes event objects back to sandbox IDs. +Updated by the compute watcher on every driver observation and by gRPC handlers during sandbox creation. Used by the compute-driver event correlator to map platform events back to sandbox IDs. + +## Observability + +Supervisor session telemetry is currently emitted as plain `tracing` events from `supervisor_session.rs` (accepted, superseded, ended, relay opened/claimed). OCSF structured logging on the gateway side is a tracked follow-up -- the `openshell-ocsf` crate needs a `GatewayContext` equivalent to the sandbox's `SandboxContext` before events like `network.activity` or `app.lifecycle` can be emitted here. Sandbox-side OCSF already covers SSH authentication, network decisions, and supervisor lifecycle. ## Error Handling - **gRPC errors**: All gRPC handlers return `tonic::Status` with appropriate codes: - - `InvalidArgument` for missing/malformed fields - - `NotFound` for nonexistent objects - - `AlreadyExists` for duplicate creation - - `FailedPrecondition` for state violations (e.g., exec on non-Ready sandbox, missing provider) - - `Internal` for store/decode/Kubernetes failures - - `PermissionDenied` for policy violations - - `ResourceExhausted` for broadcast lag (missed messages) - - `Cancelled` for closed broadcast channels - -- **HTTP errors**: The SSH tunnel handler returns HTTP status codes directly (`401`, `404`, `405`, `412`, `500`, `502`). + - `InvalidArgument` for missing/malformed fields, including a non-`Init` first frame on `RelayStream`. + - `NotFound` for nonexistent objects, including unknown or expired relay channels on `claim_relay`. + - `AlreadyExists` for duplicate creation. + - `FailedPrecondition` for state violations (e.g., exec on non-Ready sandbox, missing provider). + - `Unavailable` when the supervisor session for a sandbox is not connected within `open_relay`'s wait window, or when the supervisor's outbound channel has closed between lookup and send. + - `DeadlineExceeded` when a pending relay slot is claimed past `RELAY_PENDING_TIMEOUT`, or when `relay_rx` fails to deliver in time. + - `Internal` for store/decode/driver failures and for the `claim_relay` case where the waiter has dropped the oneshot receiver. + - `PermissionDenied` for policy violations. + - `ResourceExhausted` for broadcast lag (missed messages). + - `Cancelled` for closed broadcast channels. + +- **HTTP errors**: The SSH tunnel handler returns HTTP status codes directly (`401`, `404`, `405`, `412`, `429`, `500`, `502`). `502` indicates the supervisor relay could not be opened; `429` indicates a per-token or per-sandbox concurrent-tunnel limit. - **Connection errors**: Logged at `error` level but do not crash the gateway. TLS handshake failures and individual connection errors are caught and logged per-connection. -- **Background task errors**: The sandbox watcher and event tailer log warnings for individual processing failures but continue running. If the watcher stream ends, it logs a warning and the task exits (no automatic restart). +- **Background task errors**: The compute watcher and relay reaper log warnings for individual processing failures but continue running. If the watcher stream ends, it logs a warning and the task exits (no automatic restart). ## Cross-References -- [Sandbox Architecture](sandbox.md) -- sandbox-side policy enforcement, proxy, and isolation details +- [Sandbox Architecture](sandbox.md) -- sandbox-side policy enforcement, supervisor, and the local SSH daemon on the other end of the relay - [Gateway Settings Channel](gateway-settings.md) -- runtime settings channel, two-tier resolution, CLI/TUI commands - [Inference Routing](inference-routing.md) -- end-to-end inference interception flow, sandbox-side proxy logic, and route resolution - [Container Management](build-containers.md) -- how sandbox container images are built and configured diff --git a/architecture/sandbox-connect.md b/architecture/sandbox-connect.md index 9abb0383a1..88505c7f16 100644 --- a/architecture/sandbox-connect.md +++ b/architecture/sandbox-connect.md @@ -8,9 +8,23 @@ Sandbox connect provides secure remote access into running sandbox environments. 2. **Command execution** (`sandbox create -- `) -- runs a command over SSH with stdout/stderr piped back 3. **File sync** (`sandbox create --upload`) -- uploads local files into the sandbox before command execution -All three modes tunnel SSH traffic through the gateway's multiplexed port using HTTP CONNECT. The gateway authenticates each connection with a short-lived session token, then performs a custom NSSH1 handshake with the sandbox's embedded SSH daemon before bridging raw bytes between client and sandbox. +Gateway connectivity is **supervisor-initiated**: the gateway never dials the sandbox pod. On startup, each sandbox's supervisor opens a long-lived bidirectional gRPC stream (`ConnectSupervisor`) to the gateway and holds it for the sandbox's lifetime. When a client asks the gateway for SSH, the gateway sends a `RelayOpen` message over that stream; the supervisor responds by initiating a `RelayStream` gRPC call that rides the same TCP+TLS+HTTP/2 connection as a new multiplexed stream. The supervisor bridges the bytes of that stream into a root-owned Unix socket where the embedded SSH daemon listens. -There is also a gateway-side `ExecSandbox` gRPC RPC that executes commands inside sandboxes without requiring an external SSH client. This is used for programmatic execution. +There is also a gateway-side `ExecSandbox` gRPC RPC that executes commands inside sandboxes without requiring an external SSH client. It uses the same relay mechanism. + +## Two-Plane Architecture + +The supervisor and gateway maintain two logical planes over **one TCP+TLS connection**, multiplexed by HTTP/2 streams: + +- **Control plane** -- the `ConnectSupervisor` bidirectional gRPC stream. Carries `SupervisorHello`, heartbeats, `RelayOpen`/`RelayClose` requests from the gateway, and `RelayOpenResult`/`RelayClose` replies from the supervisor. Lives for the lifetime of the sandbox supervisor process. +- **Data plane** -- one `RelayStream` bidirectional gRPC call per SSH connect or exec invocation. Each call is a new HTTP/2 stream on the same connection. Frames are opaque bytes except for the first frame from the supervisor, which is a typed `RelayInit { channel_id }` used to pair the stream with a pending relay slot on the gateway. + +Running both planes over one HTTP/2 connection means each relay avoids a fresh TLS handshake and benefits from a single authenticated transport boundary. Hyper/h2 `adaptive_window(true)` is enabled on both sides so bulk transfers (large file uploads, long exec stdout) aren't pinned to the default 64 KiB stream window. + +The supervisor-initiated direction gives the model two properties: + +1. The sandbox pod exposes no ingress surface. Network reachability is whatever the supervisor itself can reach outward. +2. Authentication reduces to one place: the existing gateway mTLS channel. There is no second application-layer handshake to design, rotate, or replay-protect. ## Components @@ -18,70 +32,126 @@ There is also a gateway-side `ExecSandbox` gRPC RPC that executes commands insid **File**: `crates/openshell-cli/src/ssh.rs` -Contains the client-side SSH and editor-launch helpers for sandbox connectivity: +Client-side SSH and editor-launch helpers: - `sandbox_connect()` -- interactive SSH shell session - `sandbox_exec()` -- non-interactive command execution via SSH -- `sandbox_rsync()` -- file synchronization via rsync over SSH +- `sandbox_rsync()` -- file synchronization via tar-over-SSH - `sandbox_ssh_proxy()` -- the `ProxyCommand` process that bridges stdin/stdout to the gateway -- OpenShell-managed SSH config helpers -- install a single `Include` entry in - `~/.ssh/config` and maintain generated `Host openshell-` blocks in a - separate OpenShell-owned config file for editor workflows +- OpenShell-managed SSH config helpers -- install a single `Include` entry in `~/.ssh/config` and maintain generated `Host openshell-` blocks in a separate OpenShell-owned config file for editor workflows -These are re-exported from `crates/openshell-cli/src/run.rs` for backward compatibility. +Every generated SSH invocation and every entry in the OpenShell-managed `~/.ssh/config` include `ServerAliveInterval=15` and `ServerAliveCountMax=3`. SSH has no other way to observe that the underlying relay (not the end-to-end TCP socket) has silently dropped, so the client falls back to SSH-level keepalives to surface dead connections within ~45 seconds. + +These helpers are re-exported from `crates/openshell-cli/src/run.rs` for backward compatibility. ### CLI `ssh-proxy` subcommand -**File**: `crates/openshell-cli/src/main.rs` (line ~139, `Commands::SshProxy`) +**File**: `crates/openshell-cli/src/main.rs` (`Commands::SshProxy`) -A top-level CLI subcommand (`ssh-proxy`) that the SSH `ProxyCommand` invokes. It receives `--gateway`, `--sandbox-id`, and `--token` flags, then delegates to `sandbox_ssh_proxy()`. This process has no TTY of its own -- it pipes stdin/stdout directly to the gateway tunnel. +A top-level CLI subcommand (`ssh-proxy`) that the SSH `ProxyCommand` invokes. It receives `--gateway`, `--sandbox-id`, `--token`, and `--gateway-name` flags, then delegates to `sandbox_ssh_proxy()`. This process has no TTY of its own -- it pipes stdin/stdout directly to the gateway tunnel. ### gRPC session bootstrap -**Files**: `proto/openshell.proto`, `crates/openshell-server/src/grpc.rs` +**Files**: `proto/openshell.proto`, `crates/openshell-server/src/grpc/sandbox.rs` Two RPCs manage SSH session tokens: -- `CreateSshSession(sandbox_id)` -- validates the sandbox exists and is `Ready`, generates a UUID token, persists an `SshSession` record, and returns the token plus gateway connection details (host, port, scheme, connect path). +- `CreateSshSession(sandbox_id)` -- validates the sandbox exists and is `Ready`, generates a UUID token, persists an `SshSession` record, and returns the token plus gateway connection details (host, port, scheme, connect path, optional TTL). - `RevokeSshSession(token)` -- marks the session's `revoked` flag to `true` in the persistence layer. +### Supervisor session registry + +**File**: `crates/openshell-server/src/supervisor_session.rs` + +`SupervisorSessionRegistry` holds: + +- `sessions: HashMap` -- the active `ConnectSupervisor` stream sender for each sandbox, plus a `session_id` that uniquely identifies each registration. +- `pending_relays: HashMap` -- one entry per `RelayOpen` waiting for the supervisor's `RelayStream` to arrive. + +Key operations: + +- `register(sandbox_id, session_id, tx)` -- inserts a new session and returns the previous sender if it superseded one. Used by `handle_connect_supervisor` to accept a new stream. +- `remove_if_current(sandbox_id, session_id)` -- removes only if the stored `session_id` matches. Guards against the supersede race where an old session's cleanup runs after a newer session has already registered. +- `open_relay(sandbox_id, timeout)` -- called by the gateway tunnel and exec handlers. Waits up to `timeout` for a supervisor session to appear (with exponential backoff 100 ms → 2 s), registers a pending relay slot keyed by a fresh `channel_id`, sends `RelayOpen` to the supervisor, and returns a `oneshot::Receiver` that resolves when the supervisor claims the slot. +- `claim_relay(channel_id)` -- called by `handle_relay_stream` when the supervisor's first `RelayFrame::Init` arrives. Removes the pending entry, enforces a 10-second staleness bound (`RELAY_PENDING_TIMEOUT`), creates a 64 KiB `tokio::io::duplex` pair, hands the gateway-side half to the waiter, and returns the supervisor-side half to be bridged against the inbound/outbound `RelayFrame` streams. +- `reap_expired_relays()` -- bounds leaks from pending slots the supervisor never claimed (e.g., supervisor crashed between `RelayOpen` and `RelayStream`). Scheduled every 30 s by `spawn_relay_reaper()` during server startup. + +The `ConnectSupervisor` handler (`handle_connect_supervisor`) validates `SupervisorHello`, assigns a fresh `session_id`, sends `SessionAccepted { heartbeat_interval_secs: 15 }`, spawns a loop that processes inbound messages (`Heartbeat`, `RelayOpenResult`, `RelayClose`), and emits a `GatewayHeartbeat` every 15 seconds. + +### RelayStream handler + +**File**: `crates/openshell-server/src/supervisor_session.rs` (`handle_relay_stream`) + +Accepts one inbound `RelayFrame` to extract `channel_id` from `RelayInit`, claims the pending relay, then runs two concurrent forwarding tasks: + +- **Supervisor → gateway**: drains `RelayFrame::Data` frames and writes the bytes to the supervisor-side end of the duplex pair. +- **Gateway → supervisor**: reads the duplex in `RELAY_STREAM_CHUNK_SIZE` (16 KiB) chunks and emits `RelayFrame::Data` messages back. + +The first frame that isn't `RelayInit` is rejected (`invalid_argument`). Any non-data frame after init closes the relay. + ### Gateway tunnel handler **File**: `crates/openshell-server/src/ssh_tunnel.rs` An Axum route at `/connect/ssh` on the shared gateway port. Handles HTTP CONNECT requests by: -1. Validating the session token and sandbox readiness -2. Resolving the sandbox pod's network address -3. Opening a TCP connection to the sandbox SSH port -4. Performing the NSSH1 handshake -5. Bridging bytes bidirectionally between the HTTP-upgraded connection and the sandbox TCP stream + +1. Validating the session token (present, not revoked, bound to the sandbox id in `X-Sandbox-Id`, not expired). +2. Confirming the sandbox is in `Ready` phase. +3. Enforcing per-token (max 3) and per-sandbox (max 20) concurrent connection limits. +4. Calling `supervisor_sessions.open_relay(sandbox_id, 30s)` -- the 30-second wait covers the supervisor's initial mTLS + `ConnectSupervisor` handshake on a freshly-scheduled pod. +5. Waiting up to 10 seconds for the supervisor to open its `RelayStream` and deliver the gateway-side `DuplexStream`. +6. Performing the HTTP CONNECT upgrade on the client connection and calling `copy_bidirectional` between the upgraded client socket and the relay stream. + +There is no gateway-to-sandbox TCP dial, handshake preface, or pod-IP resolution in this path. ### Gateway multiplexing **File**: `crates/openshell-server/src/multiplex.rs` -The gateway runs a single listener that multiplexes gRPC and HTTP on the same port. `MultiplexedService` routes based on the `content-type` header: requests with `application/grpc` go to the gRPC router; all others (including HTTP CONNECT) go to the HTTP router. The HTTP router (`crates/openshell-server/src/http.rs`) merges health endpoints with the SSH tunnel router. +The gateway runs a single listener that multiplexes gRPC and HTTP on the same port. `MultiplexedService` routes based on the `content-type` header: requests with `application/grpc` go to the gRPC router; all others (including HTTP CONNECT) go to the HTTP router. The HTTP router (`crates/openshell-server/src/http.rs`) merges health endpoints with the SSH tunnel router. Hyper is configured with `http2().adaptive_window(true)` so the HTTP/2 stream windows grow under load rather than throttling `RelayStream` to the default 64 KiB window. + +### Sandbox supervisor session + +**File**: `crates/openshell-sandbox/src/supervisor_session.rs` + +`spawn(endpoint, sandbox_id, ssh_socket_path)` starts a background task that: + +1. Opens a gRPC `Channel` to the gateway (`http2_adaptive_window(true)`). The same channel multiplexes the control stream and every relay. +2. Sends `SupervisorHello { sandbox_id, instance_id }` as the first outbound message. +3. Waits for `SessionAccepted` (or fails fast on `SessionRejected`). +4. Runs a loop that reads inbound `GatewayMessage` values and emits `SupervisorHeartbeat` at the accepted interval (min 5 s, usually 15 s). +5. On `RelayOpen`, spawns `handle_relay_open()` which opens a new `RelayStream` RPC on the existing channel, sends `RelayInit { channel_id }` as the first frame, dials the local SSH Unix socket, and bridges bytes in both directions in 16 KiB chunks. + +Reconnect policy: the session loop wraps `run_single_session()` with exponential backoff (1 s → 30 s) on any error. A `session_established` / `session_failed` OCSF event is emitted on each attempt. + +The supervisor is a dumb byte bridge with no awareness of the SSH protocol flowing through it. ### Sandbox SSH daemon **File**: `crates/openshell-sandbox/src/ssh.rs` An embedded SSH server built on `russh` that runs inside each sandbox pod. It: -- Generates an ephemeral Ed25519 host key on startup (no persistent key material) -- Validates the NSSH1 handshake preface before starting the SSH protocol -- Accepts any SSH authentication (none or public key) since authorization is handled by the gateway -- Spawns shell processes on a PTY with full sandbox policy enforcement (Landlock, seccomp, network namespace, privilege dropping) -- Supports interactive shells, exec commands, PTY resize, and window change events + +- Generates an ephemeral Ed25519 host key on startup (no persistent key material). +- Listens on a Unix socket (default `/run/openshell/ssh.sock`, see [Unix socket access control](#unix-socket-access-control)). +- Accepts any SSH authentication (none or public key) because authorization is handled upstream by the gateway session token and by filesystem permissions on the socket. +- Spawns shell processes on a PTY with full sandbox policy enforcement (Landlock, seccomp, network namespace, privilege dropping). +- Supports interactive shells, exec commands, PTY resize, window-change events, and loopback-only `direct-tcpip` channels for port forwarding. ### Gateway-side exec (gRPC) -**File**: `crates/openshell-server/src/grpc.rs` (functions `stream_exec_over_ssh`, `start_single_use_ssh_proxy`, `run_exec_with_russh`) +**File**: `crates/openshell-server/src/grpc/sandbox.rs` (`handle_exec_sandbox`, `stream_exec_over_relay`, `start_single_use_ssh_proxy_over_relay`, `run_exec_with_russh`) The `ExecSandbox` gRPC RPC provides programmatic command execution without requiring an external SSH client. It: -1. Spins up a single-use local TCP proxy that performs the NSSH1 handshake -2. Connects a `russh` client through that proxy -3. Authenticates with `none` auth, opens a channel, sends the command -4. Streams stdout/stderr chunks and exit status back to the gRPC caller + +1. Validates `sandbox_id`, `command`, env keys, and field sizes; confirms the sandbox is `Ready`. +2. Calls `supervisor_sessions.open_relay(sandbox_id, 15s)` -- a shorter wait than connect because exec runs in steady state, not on cold start. +3. Waits up to 10 seconds for the relay `DuplexStream` to arrive. +4. Starts a single-use localhost TCP listener on `127.0.0.1:0` and spawns a task that bridges a single accept to the `DuplexStream` with `copy_bidirectional`. This adapts the `DuplexStream` to something `russh::client::connect_stream` can dial. +5. Connects `russh` to the local proxy, authenticates `none` as user `sandbox`, opens a channel, optionally requests a PTY, and executes the shell-escaped command. +6. Streams `stdout`/`stderr`/`exit` events back to the gRPC caller. + +If `timeout_seconds > 0`, the exec is wrapped in `tokio::time::timeout`. On timeout, exit code 124 is sent (matching the `timeout` command convention). ## Connection Flows @@ -93,104 +163,106 @@ The `sandbox connect` command opens an interactive SSH session. sequenceDiagram participant User as User Terminal participant CLI as CLI (sandbox connect) - participant gRPC as Gateway (gRPC) - participant Proxy as CLI (ssh-proxy) - participant GW as Gateway (/connect/ssh) - participant K8s as Pod Resolver - participant SSHD as Sandbox SSH Daemon - - CLI->>gRPC: GetSandbox(name) -> sandbox.id - CLI->>gRPC: CreateSshSession(sandbox_id) - gRPC-->>CLI: token, gateway_host, gateway_port, scheme, connect_path - - Note over CLI: Builds ProxyCommand string
exec()s into ssh process - - User->>Proxy: ssh spawns ProxyCommand subprocess - Proxy->>GW: CONNECT /connect/ssh HTTP/1.1
X-Sandbox-Id, X-Sandbox-Token - GW->>GW: Validate token + sandbox phase - GW->>K8s: Resolve pod IP (or service DNS) - GW->>SSHD: TCP connect to port 2222 - GW->>SSHD: NSSH1 preface (token, ts, nonce, hmac) - SSHD-->>GW: OK - GW-->>Proxy: 200 OK (upgrade) - - Note over Proxy,SSHD: Bidirectional byte stream (SSH protocol) - - Proxy->>SSHD: SSH handshake + auth_none - SSHD-->>Proxy: Auth accepted - Proxy->>SSHD: channel_open + shell_request + participant GW as Gateway + participant Reg as SessionRegistry + participant Sup as Supervisor (sandbox) + participant Sock as SSH Unix socket + participant SSHD as russh daemon + + Note over Sup,GW: On sandbox startup (persistent): + Sup->>GW: ConnectSupervisor stream + SupervisorHello + GW-->>Sup: SessionAccepted{session_id, heartbeat=15s} + + User->>CLI: openshell sandbox connect foo + CLI->>GW: GetSandbox(name) -> sandbox.id + CLI->>GW: CreateSshSession(sandbox_id) + GW-->>CLI: token, gateway_host, gateway_port, scheme, connect_path + + Note over CLI: Builds ProxyCommand string; exec()s ssh + + User->>CLI: ssh spawns ssh-proxy subprocess + CLI->>GW: CONNECT /connect/ssh
X-Sandbox-Id, X-Sandbox-Token + GW->>GW: Validate token + sandbox Ready + GW->>Reg: open_relay(sandbox_id, 30s) + Reg-->>GW: (channel_id, relay_rx) + GW->>Sup: RelayOpen{channel_id} (over ConnectSupervisor) + + Sup->>GW: RelayStream RPC (new HTTP/2 stream) + Sup->>GW: RelayFrame::Init{channel_id} + GW->>Reg: claim_relay(channel_id) -> DuplexStream pair + Reg-->>GW: gateway-side DuplexStream (via relay_rx) + Sup->>Sock: UnixStream::connect(/run/openshell/ssh.sock) + Sock-->>SSHD: connection accepted + + GW-->>CLI: 200 OK (upgrade) + + Note over CLI,SSHD: SSH protocol over:
CLI↔GW (HTTP CONNECT) ↔ RelayStream frames ↔ Sup ↔ Unix socket ↔ SSHD + + CLI->>SSHD: SSH handshake + auth_none + SSHD-->>CLI: Auth accepted + CLI->>SSHD: channel_open + shell_request SSHD->>SSHD: openpty() + spawn /bin/bash -i
(with sandbox policy applied) User<<->>SSHD: Interactive PTY session ``` **Code trace for `sandbox connect`:** -1. `crates/openshell-cli/src/main.rs` -- `SandboxCommands::Connect { name }` dispatches to `run::sandbox_connect()` +1. `crates/openshell-cli/src/main.rs` -- `SandboxCommands::Connect { name }` dispatches to `run::sandbox_connect()`. 2. `crates/openshell-cli/src/ssh.rs` -- `sandbox_connect()` calls `ssh_session_config()`: - - Resolves sandbox name to ID via `GetSandbox` gRPC - - Creates an SSH session via `CreateSshSession` gRPC - - Builds a `ProxyCommand` string: ` ssh-proxy --gateway --sandbox-id --token ` - - If the gateway host is loopback but the cluster endpoint is not, `resolve_ssh_gateway()` overrides the host with the cluster endpoint's host + - Resolves sandbox name to ID via `GetSandbox` gRPC. + - Creates an SSH session via `CreateSshSession` gRPC. + - Builds a `ProxyCommand` string: ` ssh-proxy --gateway --sandbox-id --token --gateway-name `. + - If the gateway host is loopback but the cluster endpoint is not, `resolve_ssh_gateway()` overrides the host with the cluster endpoint's host. 3. `sandbox_connect()` builds an `ssh` command with: - - `-o ProxyCommand=...` (the proxy command from step 2) + - `-o ProxyCommand=...` - `-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o GlobalKnownHostsFile=/dev/null` (ephemeral host keys) + - `-o ServerAliveInterval=15 -o ServerAliveCountMax=3` (surface silently-dropped relays in ~45 s) - `-tt -o RequestTTY=force` (force PTY allocation) - - `-o SetEnv=TERM=xterm-256color` (terminal type) + - `-o SetEnv=TERM=xterm-256color` - `sandbox` as the SSH user -4. If stdin is a terminal (interactive), the CLI calls `exec()` (Unix) to replace itself with the `ssh` process, giving SSH direct terminal ownership. Otherwise it spawns and waits. -5. When SSH starts, it spawns the `ssh-proxy` subprocess as its `ProxyCommand`. -6. `crates/openshell-cli/src/ssh.rs` -- `sandbox_ssh_proxy()`: - - Parses the gateway URL, connects via TCP (plain) or TLS (mTLS) - - Sends a raw HTTP CONNECT request with `X-Sandbox-Id` and `X-Sandbox-Token` headers - - Reads the response status line; proceeds if 200 - - Spawns two `tokio::spawn` tasks for bidirectional copy between stdin/stdout and the gateway stream - - When the remote-to-stdout direction completes, aborts the stdin-to-remote task (SSH has all the data it needs) +4. If stdin is a terminal (interactive), the CLI calls `exec()` (Unix) to replace itself with the `ssh` process. Otherwise it spawns and waits. +5. `sandbox_ssh_proxy()` connects via TCP (plain) or TLS (mTLS) to the gateway, sends a raw HTTP CONNECT request with `X-Sandbox-Id` and `X-Sandbox-Token` headers, and on a 200 response spawns two tasks to copy bytes between stdin/stdout and the tunnel. +6. Gateway-side: `ssh_connect()` in `ssh_tunnel.rs` authorizes the request, opens a relay, waits for the supervisor's `RelayStream`, and bridges the upgraded HTTP connection to the relay with `tokio::io::copy_bidirectional`. +7. Supervisor-side: on `RelayOpen`, `handle_relay_open()` in `crates/openshell-sandbox/src/supervisor_session.rs` opens a `RelayStream` RPC, sends `RelayInit`, dials `/run/openshell/ssh.sock`, and bridges the frames to the Unix socket. ### Command Execution (CLI) The `sandbox exec` path is identical to interactive connect except: -- The SSH command uses `-T -o RequestTTY=no` (no PTY) when `tty=false` -- The command string is passed as the final SSH argument -- The sandbox daemon routes it through `exec_request()` instead of `shell_request()`, spawning `/bin/bash -lc ` + +- The SSH command uses `-T -o RequestTTY=no` (no PTY) when `tty=false`. +- The command string is passed as the final SSH argument. +- The sandbox daemon routes it through `exec_request()` instead of `shell_request()`, spawning `/bin/bash -lc `. When `openshell sandbox create` launches a `--no-keep` command or shell, it keeps the CLI process alive instead of `exec()`-ing into SSH so it can delete the sandbox after SSH exits. The default create flow, along with `--forward`, keeps the sandbox running. ### Port Forwarding (`forward start`) -`openshell forward start ` opens a local SSH tunnel so connections to `127.0.0.1:` -on the host are forwarded to `127.0.0.1:` inside the sandbox. +`openshell forward start ` opens a local SSH tunnel so connections to `127.0.0.1:` on the host are forwarded to `127.0.0.1:` inside the sandbox. Because SSH runs over the same relay as interactive connect, no additional proxying machinery is needed. #### CLI - Reuses the same `ProxyCommand` path as `sandbox connect`. - Invokes OpenSSH with `-N -o ExitOnForwardFailure=yes -L :127.0.0.1: sandbox`. -- By default stays attached in foreground until interrupted (Ctrl+C), and prints an early startup - confirmation after SSH stays up through its initial forward-setup checks. -- With `-d`/`--background`, SSH forks after auth and the CLI exits. The PID is - tracked in `~/.config/openshell/forwards/-.pid` along with sandbox id metadata. +- By default stays attached in foreground until interrupted (Ctrl+C), and prints an early startup confirmation after SSH stays up through its initial forward-setup checks. +- With `-d`/`--background`, SSH forks after auth and the CLI exits. The PID is tracked in `~/.config/openshell/forwards/-.pid` along with sandbox id metadata. - `openshell forward stop ` validates PID ownership and then kills a background forward. - `openshell forward list` shows all tracked forwards. -- `openshell forward stop` and `openshell forward list` are local operations and do not require - resolving an active cluster. -- `openshell sandbox create --forward ` starts a background forward before connect/exec, including - when no trailing command is provided. +- `openshell forward stop` and `openshell forward list` are local operations and do not require resolving an active cluster. +- `openshell sandbox create --forward ` starts a background forward before connect/exec, including when no trailing command is provided. - `openshell sandbox delete` auto-stops any active forwards for the deleted sandbox. #### TUI -The TUI (`crates/openshell-tui/`) supports port forwarding through the create sandbox modal. Users -specify comma-separated ports in the **Ports** field. After sandbox creation: +The TUI (`crates/openshell-tui/`) supports port forwarding through the create sandbox modal. Users specify comma-separated ports in the **Ports** field. After sandbox creation: 1. The TUI polls for `Ready` state (up to 30 attempts at 2-second intervals). 2. Creates an SSH session via `CreateSshSession` gRPC. 3. Spawns background SSH tunnels (`ssh -N -f -L :127.0.0.1:`) for each port. 4. Sends a `ForwardResult` event back to the main loop with the outcome. -Active forwards are displayed in the sandbox table's NOTES column (e.g., `fwd:8080,3000`) and in -the sandbox detail view's Forwards row. +Active forwards are displayed in the sandbox table's NOTES column (e.g., `fwd:8080,3000`) and in the sandbox detail view's Forwards row. -When deleting a sandbox, the TUI calls `stop_forwards_for_sandbox()` before sending the delete -request. PID tracking uses the same `~/.config/openshell/forwards/` directory as the CLI. +When deleting a sandbox, the TUI calls `stop_forwards_for_sandbox()` before sending the delete request. PID tracking uses the same `~/.config/openshell/forwards/` directory as the CLI. #### Shared forward module @@ -202,81 +274,55 @@ Port forwarding PID management and SSH utility functions are shared between the - `save_forward_pid()` / `read_forward_pid()` / `remove_forward_pid()` -- PID file I/O - `list_forwards()` -- lists all active forwards from PID files - `stop_forward()` / `stop_forwards_for_sandbox()` -- kills forwarding processes by PID -- `resolve_ssh_gateway()` -- loopback gateway resolution (see Gateway Loopback Resolution) +- `resolve_ssh_gateway()` -- loopback gateway resolution (see [Gateway Loopback Resolution](#gateway-loopback-resolution)) - `shell_escape()` -- safe shell argument escaping for SSH commands - `build_sandbox_notes()` -- builds notes strings (e.g., `fwd:8080,3000`) from active forwards #### Supervisor `direct-tcpip` handling -The sandbox SSH server (`crates/openshell-sandbox/src/ssh.rs`) implements -`channel_open_direct_tcpip` from the russh `Handler` trait. - -- **Loopback-only**: only `127.0.0.1`, `localhost`, and `::1` destinations are accepted. - Non-loopback destinations are rejected (`Ok(false)`) to prevent the sandbox from being - used as a generic proxy. -- **Bridge**: accepted channels spawn a tokio task that connects a `TcpStream` to the - target address and uses `copy_bidirectional` between the SSH channel stream and the - TCP stream. -- No additional state is stored on `SshHandler` — the `Channel` object from russh is - self-contained, so forwarding channels are fully independent of session channels. +The sandbox SSH server (`crates/openshell-sandbox/src/ssh.rs`) implements `channel_open_direct_tcpip` from the russh `Handler` trait. -#### Flow - -```mermaid -sequenceDiagram - participant App as Local Application - participant SSH as OpenSSH Client - participant GW as Gateway (CONNECT) - participant SSHD as Sandbox SSH - participant SVC as Service in Sandbox - - SSH->>GW: CONNECT /connect/ssh - GW->>SSHD: TCP + Preface handshake - SSH->>SSHD: direct-tcpip channel (127.0.0.1:port) - SSHD->>SVC: TcpStream::connect(127.0.0.1:port) - App->>SSH: connect to 127.0.0.1:port (local) - SSH->>SSHD: channel data - SSHD->>SVC: TCP data - SVC-->>SSHD: TCP response - SSHD-->>SSH: channel data - SSH-->>App: response -``` +- **Loopback-only**: only `127.0.0.1`, `localhost`, and `::1` destinations are accepted. Non-loopback destinations are rejected (`Ok(false)`) to prevent the sandbox from being used as a generic proxy. +- **Bridge**: accepted channels spawn a tokio task that connects a `TcpStream` to the target address and uses `copy_bidirectional` between the SSH channel stream and the TCP stream. ### Gateway-side Exec (gRPC) -The `ExecSandbox` gRPC RPC bypasses the external SSH client entirely. +The `ExecSandbox` gRPC RPC bypasses the external SSH client entirely while using the same relay plumbing. ```mermaid sequenceDiagram participant Client as gRPC Client - participant Server as Gateway (gRPC) - participant Proxy as Single-Use TCP Proxy - participant SSHD as Sandbox SSH Daemon - - Client->>Server: ExecSandbox(sandbox_id, command, stdin, timeout) - Server->>Server: Validate sandbox exists + Ready - Server->>Server: Resolve target host:port - Server->>Proxy: Bind 127.0.0.1:0 (ephemeral port) - Proxy->>SSHD: TCP connect + NSSH1 handshake - SSHD-->>Proxy: OK - - Server->>Proxy: russh client connects to 127.0.0.1: - Proxy<<->>SSHD: Bridge bytes bidirectionally - Server->>SSHD: SSH auth_none + channel_open + exec(command) - Server->>SSHD: stdin payload + EOF + participant GW as Gateway + participant Reg as SessionRegistry + participant Sup as Supervisor + participant SSHD as SSH daemon (Unix socket) + + Client->>GW: ExecSandbox(sandbox_id, command, stdin, timeout) + GW->>GW: Validate sandbox exists + Ready + GW->>Reg: open_relay(sandbox_id, 15s) + Reg-->>GW: (channel_id, relay_rx) + GW->>Sup: RelayOpen{channel_id} + + Sup->>GW: RelayStream + RelayInit{channel_id} + GW->>Reg: claim_relay -> DuplexStream + Sup->>SSHD: connect /run/openshell/ssh.sock + + Note over GW: start_single_use_ssh_proxy_over_relay
(127.0.0.1:ephemeral -> DuplexStream) + + GW->>GW: russh client dials 127.0.0.1: + GW->>SSHD: SSH auth_none + channel_open + exec(command) + GW->>SSHD: stdin payload + EOF loop Stream output - SSHD-->>Server: stdout/stderr chunks - Server-->>Client: ExecSandboxEvent (Stdout/Stderr) + SSHD-->>GW: stdout/stderr chunks + GW-->>Client: ExecSandboxEvent (Stdout/Stderr) end - SSHD-->>Server: ExitStatus - Server-->>Client: ExecSandboxEvent (Exit) + SSHD-->>GW: ExitStatus + GW-->>Client: ExecSandboxEvent (Exit) ``` -The `start_single_use_ssh_proxy()` function creates a one-shot TCP listener on localhost, accepts a single connection, performs the NSSH1 handshake with the sandbox, then bridges bytes. The `run_exec_with_russh()` function connects through this local proxy, authenticates, executes the command, and streams channel messages to the gRPC response stream. - -If `timeout_seconds > 0`, the exec is wrapped in `tokio::time::timeout`. On timeout, exit code 124 is sent (matching the `timeout` command convention). +`start_single_use_ssh_proxy_over_relay()` exists only as an adapter so `russh::client::connect_stream` can consume the relay `DuplexStream` through an ephemeral TCP listener on `127.0.0.1:0`. It never reaches the network. ### File Sync @@ -288,10 +334,10 @@ File sync uses **tar-over-SSH**: the CLI streams a tar archive through the exist When `--upload` is passed to `sandbox create`, the CLI pushes local files into `/sandbox` (or a specified destination) after the sandbox reaches `Ready` and before any command runs. -1. `git_repo_root()` determines the repository root via `git rev-parse --show-toplevel` -2. `git_sync_files()` lists files with `git ls-files -co --exclude-standard -z` (tracked + untracked, respecting gitignore, null-delimited) -3. `sandbox_sync_up_files()` creates an SSH session config, spawns `ssh sandbox "tar xf - -C /sandbox"`, and streams a tar archive of the file list to the SSH child's stdin using the `tar` crate -4. Files land in `/sandbox` inside the container +1. `git_repo_root()` determines the repository root via `git rev-parse --show-toplevel`. +2. `git_sync_files()` lists files with `git ls-files -co --exclude-standard -z` (tracked + untracked, respecting gitignore, null-delimited). +3. `sandbox_sync_up_files()` creates an SSH session config, spawns `ssh sandbox "tar xf - -C /sandbox"`, and streams a tar archive of the file list to the SSH child's stdin using the `tar` crate. +4. Files land in `/sandbox` inside the container. #### `openshell sandbox upload` / `openshell sandbox download` @@ -307,151 +353,95 @@ openshell sandbox download [] - **Upload**: `sandbox_upload()` streams a tar archive of the local path to `ssh ... tar xf - -C ` on the sandbox side. Default destination: `/sandbox`. - **Download**: `sandbox_download()` runs `ssh ... tar cf - -C ` on the sandbox side and extracts the output locally via `tar::Archive`. Default destination: `.` (current directory). -- No compression for v1 — the SSH tunnel is local-network; compression adds CPU cost with marginal bandwidth savings. - -#### Why tar-over-SSH instead of rsync - -| | tar-over-SSH | rsync | -|---|---|---| -| **Client dependency** | None — `tar` crate is compiled into the CLI | Requires `rsync` installed on the client machine | -| **Sandbox dependency** | GNU `tar` (present in every base image) | Requires `rsync` installed in the container | -| **Bidirectional** | Same pipe pattern reversed for push/pull | Needs different invocation or rsync daemon for pull | -| **Transport complexity** | Single process (`ssh ... tar xf -`) | Two processes coordinating a delta-transfer protocol through the proxy tunnel | -| **Incremental sync** | Re-sends everything every time | Only transfers changed blocks (faster for repeated syncs of large repos) | -| **Compression** | Uncompressed (can add gzip via `flate2` later) | Built-in `-z` flag | +- No compression for v1 -- the SSH tunnel rides the already-TLS-encrypted gateway connection; compression adds CPU cost with marginal bandwidth savings. -For OpenShell's use case — one-shot or on-demand pushes of project files over a local network tunnel — the incremental sync advantage of rsync is marginal. Eliminating the external dependency and getting clean bidirectional support outweigh the delta-transfer benefit. If repeated rapid re-syncs of large repos become a need (e.g., a watch mode), revisit by adding content-hash-based skip lists or gzip compression. +## Supervisor Session Lifecycle -## NSSH1 Handshake Protocol +Each sandbox has at most one live `ConnectSupervisor` stream at a time. The registry enforces this via `register()`, which overwrites any previous entry. -The NSSH1 ("OpenShell SSH v1") handshake authenticates the gateway to the sandbox daemon, preventing direct pod access from outside the gateway. +### States -### Wire Format - -A single newline-terminated text line: - -``` -NSSH1 \n +```mermaid +stateDiagram-v2 + [*] --> Connecting: spawn() + Connecting --> Rejected: SessionRejected + Connecting --> Live: SessionAccepted + Live --> Live: Heartbeats
RelayOpen/Result
RelayClose + Live --> Disconnected: stream closed / error + Disconnected --> Connecting: backoff (1s..30s) + Rejected --> Connecting: backoff (1s..30s) + Live --> [*]: sandbox exits ``` -| Field | Type | Description | -|-------------|--------|-------------| -| `NSSH1` | string | Magic prefix (protocol version identifier) | -| `token` | string | UUID session token (from `CreateSshSession` for interactive; freshly generated for gateway-side exec) | -| `timestamp` | i64 | Unix epoch seconds at time of generation | -| `nonce` | string | UUID v4, unique per handshake attempt | -| `hmac` | string | Hex-encoded HMAC-SHA256 of `token\|timestamp\|nonce` keyed on the shared secret | +### Hello and accept -### Validation (sandbox side) +The supervisor sends `SupervisorHello { sandbox_id, instance_id }` (where `instance_id` is a fresh UUID per process start) as the first message. The gateway: -**File**: `crates/openshell-sandbox/src/ssh.rs` -- `verify_preface()` +1. Assigns `session_id = Uuid::new_v4()`. +2. Registers the session; any existing entry is evicted and its sender is dropped. +3. Replies with `SessionAccepted { session_id, heartbeat_interval_secs: 15 }`. +4. Spawns `run_session_loop` to process inbound messages and emit gateway heartbeats. -1. Split line on whitespace; reject if not exactly 5 fields or magic is not `NSSH1` -2. Parse timestamp; compute absolute clock skew `|now - timestamp|` -3. Reject if skew exceeds `ssh_handshake_skew_secs` (default: 300 seconds) -4. Recompute HMAC-SHA256 over `token|timestamp|nonce` with the shared secret -5. Compare computed signature against the received signature (constant-time via `hmac` crate) -6. Check nonce against the replay cache; reject if the nonce has been seen before within the skew window -7. Insert the nonce into the replay cache on success -8. Respond with `OK\n` on success or `ERR\n` on failure +On any registration failure (e.g., the supervisor's mpsc receiver was already dropped), `remove_if_current` is called with the assigned `session_id` so the cleanup does not evict a newer successful registration. -### Nonce replay detection +### Heartbeats -The SSH server maintains a per-process `NonceCache` (`HashMap` behind `Arc>`) that tracks nonces seen within the handshake skew window. A background tokio task reaps expired entries every 60 seconds. If a valid preface is presented with a previously-seen nonce, the handshake is rejected. This prevents replay attacks within the timestamp validity window. - -### HMAC computation - -Both the gateway (`crates/openshell-server/src/ssh_tunnel.rs` -- `build_preface()`) and the gRPC exec path (`crates/openshell-server/src/grpc.rs` -- `build_preface()`) use identical logic: - -```rust -let payload = format!("{token}|{timestamp}|{nonce}"); -let signature = hmac_sha256(secret.as_bytes(), payload.as_bytes()); -// hmac_sha256 returns hex::encode(Hmac::::finalize()) -``` +Both directions emit heartbeats at the negotiated interval (15 s). Heartbeats are strictly informational -- their purpose is to keep the HTTP/2 connection warm and let each side detect a half-open transport quickly. There is no explicit application-level timeout that kills the session if heartbeats stop; failures are detected when a send fails or when the stream reports EOF / error. -### Read-line safety +### Supersede semantics -Both sides cap the preface line at 1024 bytes and stop reading at `\n` or EOF. This prevents a misbehaving peer from consuming unbounded memory. +If a supervisor restarts (or a network blip forces a new `ConnectSupervisor` call), the gateway sees a second `SupervisorHello` for the same `sandbox_id`. `register()` inserts the new session and returns the old `tx`. The old session's `run_session_loop` continues to poll its inbound stream until it errors out, at which point its cleanup calls `remove_if_current(sandbox_id, old_session_id)` -- which does nothing because the stored entry now has the new `session_id`. The newer session stays live. -## Sandbox SSH Daemon Internals +Tests in `supervisor_session.rs` pin this behavior: -### Startup +- `registry_supersedes_previous_session` -- confirms that `register()` returns the prior sender. +- `remove_if_current_ignores_stale_session_id` -- confirms a late cleanup does not evict a newer registration. +- `open_relay_uses_newest_session_after_supersede` -- confirms `RelayOpen` is delivered to the newest session only. -`run_ssh_server()` in `crates/openshell-sandbox/src/ssh.rs`: +### Pending-relay reaper -1. Generates an ephemeral Ed25519 host key using `OsRng` -2. Configures `russh::server::Config` with 1-second auth rejection delay -3. Binds a `TcpListener` on the configured address (default: `0.0.0.0:2222`) -4. Enters an accept loop; each connection is handled in a `tokio::spawn` task +`spawn_relay_reaper(state, 30s)` sweeps `pending_relays` every 30 seconds and removes entries older than `RELAY_PENDING_TIMEOUT` (10 s). This bounds the leak if a supervisor acknowledges `RelayOpen` but crashes before initiating `RelayStream`. -### Connection handling +## Authentication and Security Model -`handle_connection()`: +### Transport authentication -1. Reads and validates the NSSH1 preface (rejects with `ERR\n` on failure) -2. Responds `OK\n` on success -3. Hands the TCP stream to `russh::server::run_stream()` with an `SshHandler` +All gRPC traffic (control plane + data plane + other RPCs) rides one mTLS-authenticated TCP+TLS+HTTP/2 connection from the supervisor to the gateway. Client certificates prove the supervisor's identity; the server certificate proves the gateway's. Nothing sits between the supervisor and the SSH daemon except the Unix socket's filesystem permissions. -### Authentication +The CLI continues to authenticate to the gateway with its own mTLS credentials (or Cloudflare bearer token in reverse-proxy deployments) and a per-session token returned by `CreateSshSession`. The session token is enforced at the gateway: token scope (sandbox id), revocation state, and optional expiry are all checked in `ssh_connect()` before `open_relay()` is called. -The `SshHandler` implements `russh::server::Handler`: +### Unix socket access control -- `auth_none()` returns `Auth::Accept` -- any user is accepted -- `auth_publickey()` returns `Auth::Accept` -- any key is accepted +The supervisor creates `/run/openshell/ssh.sock` (path is configurable via the gateway's `sandbox_ssh_socket_path` / supervisor's `--ssh-socket-path` / `OPENSHELL_SSH_SOCKET_PATH`) and: -Authorization is performed by the gateway (token validation + sandbox readiness check) before the SSH protocol starts. The NSSH1 handshake proves the connection came through an authorized gateway. +1. Creates the parent directory if missing and sets it to mode `0700` (root-owned). +2. Removes any stale socket from a previous run. +3. Binds a `UnixListener` on the path. +4. Sets the socket to mode `0600`. -### Shell and exec +The supervisor runs as root; the sandbox workload runs as an unprivileged user. Only the supervisor can connect to the socket. The workload inside the sandbox has no filesystem path by which it can reach the SSH daemon directly. All ingress goes through the relay bridge, which only the supervisor can open (because only the supervisor holds the gateway session). -- `shell_request()` calls `start_shell(channel, handle, None)` -- spawns `/bin/bash -i` -- `exec_request()` calls `start_shell(channel, handle, Some(command))` -- spawns `/bin/bash -lc ` -- `pty_request()` stores the PTY dimensions for use when spawning the shell -- `window_change_request()` calls `TIOCSWINSZ` ioctl on the PTY master fd +`handle_connection()` in `crates/openshell-sandbox/src/ssh.rs` hands the Unix stream directly to `russh::server::run_stream` with no preface or handshake layer in between. -### PTY and process management +### Kubernetes NetworkPolicy -`spawn_pty_shell()`: +The sandbox pod needs no gateway-to-sandbox ingress rule; the SSH daemon has no TCP listener. Helm ships an egress policy that constrains what the pod can reach outward -- see [Gateway Security](gateway-security.md). -1. Calls `nix::pty::openpty()` with the requested window size -2. Clones the master fd for reading and writing -3. Configures the shell command with environment variables: - - `OPENSHELL_SANDBOX=1`, `HOME=/sandbox`, `USER=sandbox`, `TERM=` - - Proxy vars: `HTTP_PROXY`, `HTTPS_PROXY`, `ALL_PROXY`, `NO_PROXY=127.0.0.1,localhost,::1`, `http_proxy`, `https_proxy`, `grpc_proxy`, `no_proxy=127.0.0.1,localhost,::1`, `NODE_USE_ENV_PROXY=1` so Node.js `fetch` honors the proxy env while localhost stays direct - - TLS trust vars: `NODE_EXTRA_CA_CERTS`, `SSL_CERT_FILE`, `REQUESTS_CA_BUNDLE`, `CURL_CA_BUNDLE` - - Provider credential env vars (from the provider registry) -4. Installs a `pre_exec` hook that: - - Calls `setsid()` to create a new session - - Calls `TIOCSCTTY` to set the slave PTY as the controlling terminal - - Enters the network namespace (`setns(fd, CLONE_NEWNET)`) if configured (Linux only) - - Drops privileges (`initgroups` + `setgid` + `setuid`) per the sandbox policy - - Applies sandbox restrictions (Landlock, seccomp) via `sandbox::apply()` -5. Spawns the child process +### What SSH auth does NOT enforce -### I/O threading +The embedded SSH daemon accepts all authentication attempts. This is intentional: -Three threads handle the PTY I/O: +- The gateway already validated the session token and sandbox readiness. +- Unix socket permissions already restrict who can connect to the daemon to the supervisor, and the supervisor only opens the socket in response to a gateway `RelayOpen`. +- SSH key management would add complexity without additional security value in this architecture. -1. **Writer thread** (std::thread) -- receives bytes from `SshHandler::data()` via an `mpsc::channel` and writes them to the PTY master -2. **Reader thread** (std::thread) -- reads from PTY master in 4096-byte chunks, dispatches each chunk to the SSH channel via `handle.data()` on the tokio runtime. Sends EOF when the master returns 0 or errors. Signals completion via a `reader_done_tx` channel. -3. **Exit thread** (std::thread) -- waits for `child.wait()`, then waits for the reader thread to finish (via `reader_done_rx`), then sends `exit_status_request` and `close` on the SSH channel +### Ephemeral host keys -The reader-done synchronization ensures correct SSH protocol ordering: data -> EOF -> exit-status -> close. +The sandbox generates a fresh Ed25519 host key on every startup. The CLI disables `StrictHostKeyChecking` and sets `UserKnownHostsFile=/dev/null` and `GlobalKnownHostsFile=/dev/null` to avoid known-hosts conflicts. ## Sandbox Target Resolution -The gateway and the gRPC exec path both resolve the sandbox's network address using the same logic. - -**File**: `crates/openshell-server/src/ssh_tunnel.rs` (gateway), `crates/openshell-server/src/grpc.rs` (exec) - -Resolution order: -1. If the sandbox has a `status.agent_pod` field, resolve the pod IP via the Kubernetes API (`agent_pod_ip()`) -2. Otherwise, construct a cluster-internal DNS name: `..svc.cluster.local` - -The target port is always `config.sandbox_ssh_port` (default: 2222). - -The `ConnectTarget` enum in `ssh_tunnel.rs` encodes both cases: -- `ConnectTarget::Ip(SocketAddr)` -- direct IP from pod resolution -- `ConnectTarget::Host(String, u16)` -- DNS hostname fallback +The gateway does not resolve a sandbox's network address or port. The only identifier that matters is `sandbox_id`, which keys into the supervisor session registry. ## API and Persistence @@ -460,9 +450,11 @@ The `ConnectTarget` enum in `ssh_tunnel.rs` encodes both cases: **Proto**: `proto/openshell.proto` -- `CreateSshSessionRequest` / `CreateSshSessionResponse` Request: + - `sandbox_id` (string) -- the sandbox to connect to Response: + - `sandbox_id` (string) - `token` (string) -- UUID session token - `gateway_host` (string) -- resolved from `Config::ssh_gateway_host` (defaults to bind address if empty) @@ -470,13 +462,16 @@ Response: - `gateway_scheme` (string) -- `"https"` if TLS is configured, otherwise `"http"` - `connect_path` (string) -- from `Config::ssh_connect_path` (default: `/connect/ssh`) - `host_key_fingerprint` (string) -- currently unused (empty) +- `expires_at_ms` (int64) -- session expiry; 0 disables expiry ### RevokeSshSession Request: + - `token` (string) -- session token to revoke Response: + - `revoked` (bool) -- true if a session was found and revoked ### SshSession persistence @@ -493,25 +488,53 @@ Stored in the gateway's persistence layer (SQLite or Postgres) as object type `" | `created_at_ms` | int64 | Creation time (ms since epoch) | | `revoked` | bool | Whether the session has been revoked | | `name` | string | Auto-generated human-friendly name | +| `expires_at_ms` | int64 | Expiry timestamp; 0 means no expiry | + +A background reaper (`spawn_session_reaper`) deletes revoked and expired rows every hour. + +### ConnectSupervisor / RelayStream + +**Proto**: `proto/openshell.proto` + +- `ConnectSupervisor(stream SupervisorMessage) returns (stream GatewayMessage)` +- `RelayStream(stream RelayFrame) returns (stream RelayFrame)` + +Key messages: + +| Message | Direction | Fields | +|---|---|---| +| `SupervisorHello` | sup → gw | `sandbox_id`, `instance_id` | +| `SessionAccepted` | gw → sup | `session_id`, `heartbeat_interval_secs` | +| `SessionRejected` | gw → sup | `reason` | +| `SupervisorHeartbeat` | sup → gw | (empty) | +| `GatewayHeartbeat` | gw → sup | (empty) | +| `RelayOpen` | gw → sup | `channel_id` (UUID) | +| `RelayOpenResult` | sup → gw | `channel_id`, `success`, `error` | +| `RelayClose` | either | `channel_id`, `reason` | +| `RelayInit` | sup → gw (first `RelayFrame`) | `channel_id` | +| `RelayFrame` | either | `oneof { RelayInit init, bytes data }` | ### ExecSandbox **Proto**: `proto/openshell.proto` -- `ExecSandboxRequest` / `ExecSandboxEvent` Request: + - `sandbox_id` (string) - `command` (repeated string) -- command and arguments - `workdir` (string) -- optional working directory - `environment` (map) -- optional env var overrides (keys validated against `^[A-Za-z_][A-Za-z0-9_]*$`) - `timeout_seconds` (uint32) -- 0 means no timeout - `stdin` (bytes) -- optional stdin payload +- `tty` (bool) -- request a PTY Response stream (`ExecSandboxEvent`): + - `Stdout(data)` -- stdout chunk - `Stderr(data)` -- stderr chunk - `Exit(exit_code)` -- final exit status (124 on timeout) -The gateway builds the remote command by shell-escaping arguments, prepending sorted env var assignments, and optionally wrapping in `cd && ...`. +The gateway builds the remote command by shell-escaping arguments, prepending sorted env var assignments, and optionally wrapping in `cd && ...`. The assembled command is capped at 256 KiB. ## Gateway Loopback Resolution @@ -523,98 +546,98 @@ The override only applies if the cluster endpoint itself is not also a loopback This function is shared between the CLI and TUI via the `openshell-core::forward` module. -## Authentication and Security Model - -### Layered authentication - -1. **mTLS (transport layer)** -- when TLS is configured, the CLI authenticates to the gateway using client certificates. The `ssh-proxy` subprocess inherits TLS options from the parent CLI process. -2. **Session token (application layer)** -- the gateway validates the session token against the persistence layer. Tokens are scoped to a specific sandbox and can be revoked. -3. **NSSH1 handshake (gateway-to-sandbox)** -- the shared handshake secret proves the connection originated from an authorized gateway. The timestamp + nonce prevent replay attacks within the skew window. The nonce replay cache rejects duplicates. -4. **Kubernetes NetworkPolicy** -- a Helm-managed `NetworkPolicy` restricts ingress to sandbox pods on port 2222 to only the gateway pod, preventing lateral movement from other in-cluster workloads. Controlled by `networkPolicy.enabled` in the Helm values (default: `true`). - -### Mandatory handshake secret - -The NSSH1 handshake secret (`OPENSHELL_SSH_HANDSHAKE_SECRET`) is required. Both the server and sandbox will refuse to start if the secret is empty or unset. For cluster deployments the secret is auto-generated by the entrypoint script (`deploy/docker/cluster-entrypoint.sh`) via `openssl rand -hex 32` and injected into the Helm values. - -### What SSH auth does NOT enforce - -The embedded SSH daemon accepts all authentication attempts. This is intentional: -- The NSSH1 handshake already proved the connection came through the gateway -- The gateway already validated the session token and sandbox readiness -- SSH key management would add complexity without additional security value in this architecture - -### Ephemeral host keys - -The sandbox generates a fresh Ed25519 host key on every startup. The CLI disables `StrictHostKeyChecking` and sets `UserKnownHostsFile=/dev/null` and `GlobalKnownHostsFile=/dev/null` to avoid known-hosts conflicts. - -## Configuration Reference - -### Gateway configuration - -**File**: `crates/openshell-core/src/config.rs` -- `Config` struct - -| Field | Default | Description | -|----------------------------|------------------|-------------| -| `ssh_gateway_host` | `127.0.0.1` | Public hostname/IP for gateway connections | -| `ssh_gateway_port` | `8080` | Public port for gateway connections (0 = use bind port) | -| `ssh_connect_path` | `/connect/ssh` | HTTP path for CONNECT requests | -| `sandbox_ssh_port` | `2222` | SSH listen port inside sandbox pods | -| `ssh_handshake_secret` | (required) | Shared HMAC key for NSSH1 handshake (server fails to start if empty) | -| `ssh_handshake_skew_secs` | `300` | Maximum allowed clock skew (seconds) | - -### Sandbox environment variables - -These are injected into sandbox pods by the gateway: +## Timeouts -| Variable | Description | -|--------------------------------------|-------------| -| `OPENSHELL_SSH_LISTEN_ADDR` | Address for the embedded SSH server to bind | -| `OPENSHELL_SSH_HANDSHAKE_SECRET` | Shared secret for NSSH1 handshake validation | -| `OPENSHELL_SSH_HANDSHAKE_SKEW_SECS` | Allowed clock skew for handshake timestamp | - -### CLI TLS options - -| Flag / Env Var | Description | -|-----------------------------|-------------| -| `--tls-ca` / `OPENSHELL_TLS_CA` | CA certificate for gateway verification | -| `--tls-cert` / `OPENSHELL_TLS_CERT` | Client certificate for mTLS | -| `--tls-key` / `OPENSHELL_TLS_KEY` | Client private key for mTLS | +| Stage | Duration | Where | +|---|---|---| +| Supervisor session wait (SSH connect) | 30 s | `ssh_tunnel::ssh_connect` -> `open_relay` | +| Supervisor session wait (ExecSandbox) | 15 s | `handle_exec_sandbox` -> `open_relay` | +| Wait for supervisor to claim relay | 10 s | `relay_rx` wrapped in `tokio::time::timeout` | +| Pending-relay TTL (reaper) | 10 s | `RELAY_PENDING_TIMEOUT` in registry | +| Session-wait backoff | 100 ms → 2 s | `wait_for_session` | +| Supervisor reconnect backoff | 1 s → 30 s | `run_session_loop` in sandbox supervisor | +| SSH-level keepalive | 15 s × 3 | CLI / managed ssh-config | +| Supervisor heartbeat | 15 s | `HEARTBEAT_INTERVAL_SECS` | +| SSH session reaper sweep | 1 h | `spawn_session_reaper` | +| Pending-relay reaper sweep | 30 s | `spawn_relay_reaper` | ## Failure Modes | Scenario | Status / Behavior | Source | -|----------|-------------------|--------| +|---|---|---| | Missing `X-Sandbox-Id` or `X-Sandbox-Token` header | `401 Unauthorized` | `ssh_tunnel.rs` -- `header_value()` | | Empty header value | `400 Bad Request` | `ssh_tunnel.rs` -- `header_value()` | | Non-CONNECT method on `/connect/ssh` | `405 Method Not Allowed` | `ssh_tunnel.rs` -- `ssh_connect()` | | Token not found in persistence | `401 Unauthorized` | `ssh_tunnel.rs` -- `ssh_connect()` | | Token revoked or sandbox ID mismatch | `401 Unauthorized` | `ssh_tunnel.rs` -- `ssh_connect()` | +| Token expired | `401 Unauthorized` | `ssh_tunnel.rs` -- `ssh_connect()` | | Sandbox not found | `404 Not Found` | `ssh_tunnel.rs` -- `ssh_connect()` | | Sandbox not in `Ready` phase | `412 Precondition Failed` | `ssh_tunnel.rs` -- `ssh_connect()` | -| Pod IP resolution fails | `502 Bad Gateway` | `ssh_tunnel.rs` -- `ssh_connect()` | -| No pod IP and no sandbox name | `412 Precondition Failed` | `ssh_tunnel.rs` -- `ssh_connect()` | -| Persistence read error | `500 Internal Server Error` | `ssh_tunnel.rs` -- `ssh_connect()` | -| NSSH1 handshake rejected by sandbox | Tunnel closed; `"sandbox handshake rejected"` logged | `ssh_tunnel.rs` -- `handle_tunnel()` | -| HTTP upgrade failure | `"SSH upgrade failed"` logged; tunnel not established | `ssh_tunnel.rs` -- `ssh_connect()` | -| TCP connection to sandbox fails | Tunnel error logged and closed | `ssh_tunnel.rs` -- `handle_tunnel()` | -| SSH exec timeout | Exit code 124 returned | `grpc.rs` -- `stream_exec_over_ssh()` | +| Per-token or per-sandbox concurrency limit hit | `429 Too Many Requests` | `ssh_tunnel.rs` -- `ssh_connect()` | +| Supervisor session not connected after 30 s | `502 Bad Gateway` | `ssh_tunnel.rs` -- `ssh_connect()` | +| Supervisor failed to claim relay within 10 s | Tunnel closed; `"relay open timed out"` logged | `ssh_tunnel.rs` -- spawned tunnel task | +| Relay channel oneshot dropped | Tunnel closed; `"relay channel dropped"` logged | `ssh_tunnel.rs` -- spawned tunnel task | +| First `RelayFrame` not `RelayInit` or empty `channel_id` | `invalid_argument` on `RelayStream` | `supervisor_session.rs` -- `handle_relay_stream` | +| `RelayStream` arrives after pending entry expired (>10 s) | `deadline_exceeded` | `supervisor_session.rs` -- `claim_relay` | +| Gateway restart during live relay | CLI SSH detects via keepalive within ~45 s; relays are torn down with the TCP connection | CLI `ServerAliveInterval=15`, `ServerAliveCountMax=3` | +| Supervisor restart | Gateway sends on stale mpsc fails; client sees same behavior as gateway restart; supervisor's reconnect loop re-registers | `run_session_loop`, `open_relay` | +| Silently-dropped relay (half-open TCP) | CLI-side SSH keepalives probe every 15 s; session exits with `Broken pipe` after 3 missed probes | SSH client keepalives | +| ExecSandbox timeout | Exit code 124 returned to caller | `stream_exec_over_relay` | +| Command exceeds 256 KiB assembled length | `invalid_argument` | `build_remote_exec_command` | ## Graceful Shutdown ### Gateway tunnel teardown -After `copy_bidirectional` completes (either side closes), `handle_tunnel()` calls `AsyncWriteExt::shutdown()` on the upgraded connection to send a clean EOF to the client. This avoids TCP RST and gives SSH time to read remaining protocol data (e.g., exit-status) from its buffer. +After `copy_bidirectional` completes on either side, `ssh_connect()` calls `AsyncWriteExt::shutdown()` on the upgraded client connection so SSH sees a clean EOF and can read any remaining protocol data (e.g., exit-status) before exiting. + +### RelayStream teardown + +The `handle_relay_stream` task half-closes the supervisor-side duplex on inbound EOF so the gateway-side reader sees EOF and terminates its own forwarding task. On the supervisor side, `handle_relay_open` does the symmetric shutdown on the Unix socket after inbound EOF, then drops the outbound mpsc so the gateway observes EOF on the response stream too. -### SSH proxy teardown +### Supervisor session teardown -The `sandbox_ssh_proxy()` function spawns two copy tasks. When the remote-to-stdout task completes, the stdin-to-remote task is aborted. This ensures the proxy exits promptly when the SSH session ends without waiting for the user to type something. +When the sandbox exits, the supervisor process ends, the HTTP/2 connection closes, and all multiplexed streams fail with `stream error`. The gateway's `run_session_loop` observes the error, logs `supervisor session: ended`, and calls `remove_if_current` to deregister. Pending relay slots that never got claimed are swept by `reap_expired_relays` within 30 s. ### PTY reader-exit ordering The sandbox SSH daemon's exit thread waits for the reader thread to finish forwarding all PTY output before sending `exit_status_request` and `close`. This prevents a race where the channel closes before all output has been delivered. +## Configuration Reference + +### Gateway configuration + +**File**: `crates/openshell-core/src/config.rs` -- `Config` struct + +| Field | Default | Description | +|---|---|---| +| `ssh_gateway_host` | `127.0.0.1` | Public hostname/IP advertised in `CreateSshSessionResponse` | +| `ssh_gateway_port` | `8080` | Public port for gateway connections (0 = use bind port) | +| `ssh_connect_path` | `/connect/ssh` | HTTP path for CONNECT requests | +| `sandbox_ssh_socket_path` | `/run/openshell/ssh.sock` | Path the supervisor binds its Unix socket on; passed to the sandbox as `OPENSHELL_SSH_SOCKET_PATH` | +| `ssh_session_ttl_secs` | (default in code) | Default TTL applied to new `SshSession` rows; 0 disables expiry | + +### Sandbox environment variables + +These are injected into sandbox pods by the Kubernetes driver (`crates/openshell-driver-kubernetes/src/driver.rs`): + +| Variable | Description | +|---|---| +| `OPENSHELL_SSH_SOCKET_PATH` | Filesystem path for the embedded SSH server's Unix socket (default `/run/openshell/ssh.sock`) | +| `OPENSHELL_ENDPOINT` | Gateway gRPC endpoint; the supervisor uses this to open `ConnectSupervisor` | +| `OPENSHELL_SANDBOX_ID` | Identifier reported in `SupervisorHello` | + +### CLI TLS options + +| Flag / Env Var | Description | +|---|---| +| `--tls-ca` / `OPENSHELL_TLS_CA` | CA certificate for gateway verification | +| `--tls-cert` / `OPENSHELL_TLS_CERT` | Client certificate for mTLS | +| `--tls-key` / `OPENSHELL_TLS_KEY` | Client private key for mTLS | + ## Cross-References - [Gateway Architecture](gateway.md) -- gateway multiplexing, persistence layer, gRPC service details +- [Gateway Security](gateway-security.md) -- mTLS, session tokens, network policy - [Sandbox Architecture](sandbox.md) -- sandbox lifecycle, policy enforcement, network isolation, proxy - [Providers](sandbox-providers.md) -- provider credential injection into SSH shell processes diff --git a/architecture/sandbox.md b/architecture/sandbox.md index fcef69b4fa..2015702f82 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -15,7 +15,8 @@ All paths are relative to `crates/openshell-sandbox/src/`. | `opa.rs` | OPA/Rego policy engine using `regorus` crate -- network evaluation, sandbox config queries, L7 endpoint queries | | `process.rs` | `ProcessHandle` for spawning child processes, privilege dropping, signal handling | | `proxy.rs` | HTTP CONNECT proxy with OPA evaluation, process-identity binding, inference interception, and L7 dispatch | -| `ssh.rs` | Embedded SSH server (`russh` crate) with PTY support and handshake verification | +| `ssh.rs` | Embedded SSH server (`russh` crate) listening on a Unix socket, with PTY support | +| `supervisor_session.rs` | Persistent outbound `ConnectSupervisor` gRPC session to the gateway; bridges `RelayStream` calls to the local SSH daemon's Unix socket | | `identity.rs` | `BinaryIdentityCache` -- SHA256 trust-on-first-use binary integrity | | `procfs.rs` | `/proc` filesystem reading for TCP peer identity resolution and ancestor chain walking | | `grpc_client.rs` | gRPC client for fetching policy, provider environment, inference route bundles, policy polling/status reporting, proposal submission, and log push (`CachedOpenShellClient`) | @@ -65,9 +66,12 @@ flowchart TD L --> L2[Spawn bypass monitor] L2 --> N{SSH enabled?} M --> N - N -- Yes --> O[Spawn SSH server task] - N -- No --> P[Spawn child process] - O --> P + N -- Yes --> O[Spawn SSH server task on Unix socket] + N -- No --> P0{gRPC mode + socket?} + O --> P0 + P0 -- Yes --> P1[Spawn supervisor session task] + P0 -- No --> P[Spawn child process] + P1 --> P P --> Q[Store entrypoint PID] Q --> R{gRPC mode?} R -- Yes --> T[Spawn policy poll task] @@ -110,7 +114,9 @@ flowchart TD - Build `InferenceContext` via `build_inference_context()` which resolves routes from one of two sources (see [Inference routing context](#inference-routing-context) below) - `ProxyHandle::start_with_bind_addr()` binds a `TcpListener` and spawns an accept loop, passing the inference context to each connection handler -8. **SSH server** (optional): If `--ssh-listen-addr` is provided, spawn an async task running `ssh::run_ssh_server()` with the policy, workdir, netns FD, proxy URL, CA paths, and provider env. +8. **SSH server** (optional): If `--ssh-socket-path` is provided, spawn an async task running `ssh::run_ssh_server()` with the policy, workdir, netns FD, proxy URL, CA paths, and provider env. The value is a filesystem path to the Unix socket the embedded sshd binds. The supervisor waits on a readiness `oneshot` channel before proceeding so that exec requests arriving immediately after pod-ready cannot race against socket bind. + +8a. **Supervisor session** (gRPC mode + SSH socket only): If `--sandbox-id`, `--openshell-endpoint`, and an SSH socket path are all set, spawn `supervisor_session::spawn()`. This task opens a persistent outbound bidirectional gRPC stream to the gateway and bridges inbound relay requests to the local SSH daemon. See [Supervisor Session](#supervisor-session) for the full protocol. 9. **Child process spawning** (`ProcessHandle::spawn()`): - Build `tokio::process::Command` with inherited stdio and `kill_on_drop(true)` @@ -1314,30 +1320,24 @@ Exit code is `code` if the process exited normally, or `128 + signal` if killed **File:** `crates/openshell-sandbox/src/ssh.rs` -The embedded SSH server provides remote shell access to the sandbox. It uses the `russh` crate and allocates PTYs for interactive sessions. +The embedded SSH server provides remote shell access to the sandbox. It uses the `russh` crate and allocates PTYs for interactive sessions. The daemon listens on a **Unix domain socket** rather than a TCP port -- the gateway never dials the sandbox pod directly. All SSH traffic arrives through the [supervisor session](#supervisor-session)'s `RelayStream` RPC, which the supervisor bridges into the socket. ### Startup -`run_ssh_server()`: -1. Generate an ephemeral Ed25519 host key via `russh::keys::PrivateKey::random()` -2. Bind a `TcpListener` to the configured address -3. Accept connections in a loop, spawning per-connection handlers - -### Handshake verification +`ssh_server_init()` (called from `run_ssh_server()`): -Before the SSH protocol begins, the server reads a preface line: +1. Generate an ephemeral Ed25519 host key via `russh::keys::PrivateKey::random()`. +2. Ensure the socket's parent directory exists and is owned by root with mode `0700`. The sandbox entrypoint runs as an unprivileged user, so it cannot enter this directory. +3. Remove any stale socket file from a prior run, then `UnixListener::bind(listen_path)`. +4. Set the socket file's mode to `0600` so only the supervisor (root) can connect to it. +5. Signal readiness back to `lib.rs` via a `oneshot` channel. +6. Accept connections in a loop and spawn `handle_connection()` per connection. -``` -NSSH1 {token} {timestamp} {nonce} {hmac_hex}\n -``` +The socket path is taken from `--ssh-socket-path` / `OPENSHELL_SSH_SOCKET_PATH`. The Kubernetes compute driver sets this to `/run/openshell/ssh.sock` by default (see `crates/openshell-driver-kubernetes/src/main.rs`); the VM driver pins it to the same path inside the guest. -`verify_preface()`: -1. Verify magic is `NSSH1` and exactly 5 fields -2. Verify `|now - timestamp|` is within `--ssh-handshake-skew-secs` (default 300s) -3. Compute `HMAC-SHA256(secret, "{token}|{timestamp}|{nonce}")` and compare with `{hmac_hex}` -4. Send `OK\n` on success, `ERR\n` on failure +### Access control -This pre-SSH handshake authenticates the gateway-to-sandbox tunnel. After it succeeds, the SSH session uses permissive authentication (`auth_none` and `auth_publickey` both return `Accept`) since the transport is already verified. +The filesystem permissions on the parent directory (`0700`) and the socket itself (`0600`) are the sole authentication boundary. Only the supervisor, which runs as root inside the container, can open the socket. The sandboxed entrypoint process -- dropped to the unprivileged `sandbox` user and further constrained by Landlock -- cannot reach `/run/openshell/` at all. Consequently, the SSH session handler's `auth_none` and `auth_publickey` callbacks both return `Auth::Accept` unconditionally; any byte stream that reaches the daemon has already passed the trust check via the socket's permission bits. ### Shell/exec handling @@ -1365,6 +1365,90 @@ The `SshHandler` implements `russh::server::Handler`: - **Reader thread**: Reads from PTY master, sends SSH channel data, sends EOF when done, signals the exit thread - **Exit thread**: Waits for child to exit, waits for reader to finish (ensures correct SSH protocol ordering: data -> EOF -> exit-status -> close), sends exit status and closes the channel +## Supervisor Session + +**File:** `crates/openshell-sandbox/src/supervisor_session.rs` + +The sandbox pod has no inbound network surface. Instead, the supervisor opens a single persistent outbound gRPC stream to the gateway and the gateway uses that stream to request on-demand byte relays back into the sandbox. All SSH connect traffic and `ExecSandbox` calls ride this connection -- there is no reverse HTTP CONNECT, no TCP listener on the pod, and no per-session TLS handshake. + +### Connection model + +```mermaid +sequenceDiagram + participant S as Supervisor (sandbox) + participant GW as Gateway + participant SSHD as Local sshd (Unix socket) + participant Client as Operator / CLI + + S->>GW: ConnectSupervisor stream (mTLS, HTTP/2) + S->>GW: SupervisorMessage::Hello{sandbox_id, instance_id} + GW-->>S: GatewayMessage::SessionAccepted{session_id, heartbeat_interval_secs} + loop Heartbeats (max(accepted.heartbeat_interval_secs, 5)) + S-->>GW: SupervisorHeartbeat + GW-->>S: GatewayHeartbeat + end + + Client->>GW: sandbox connect / ExecSandbox + GW-->>S: GatewayMessage::RelayOpen{channel_id} + S->>GW: RelayStream RPC (new HTTP/2 stream on same Channel) + S->>GW: RelayFrame::Init{channel_id} + S->>SSHD: UnixStream::connect(ssh_socket_path) + loop Relay active + GW-->>S: RelayFrame::Data (raw SSH bytes from operator) + S->>SSHD: write_all + SSHD-->>S: read chunk (up to 16 KiB) + S-->>GW: RelayFrame::Data + end +``` + +One TCP+TLS+HTTP/2 connection carries both the long-lived control stream and every concurrent relay. The sandbox-side `Endpoint` uses `adaptive_window(true)` so HTTP/2 flow control does not throttle bulk transfers (SFTP, `sandbox rsync`) to the 64 KiB default window. + +### Session lifecycle + +`spawn(endpoint, sandbox_id, ssh_socket_path)` launches `run_session_loop()`, which runs for the lifetime of the supervisor: + +1. **Connect**: `grpc_client::connect_channel_pub(endpoint)` builds an mTLS `tonic::transport::Channel`. The same `Channel` is cloned into every subsequent `RelayStream` call so no additional TLS handshakes occur. +2. **Hello**: The supervisor sends `SupervisorMessage::Hello { sandbox_id, instance_id }` as the first envelope, where `instance_id` is a fresh UUID per session. The gateway uses the sandbox ID and instance ID to supersede a stale prior session (see [Supersede](#session-supersede)). +3. **Wait for `SessionAccepted` / `SessionRejected`**: If rejected, the loop returns an error and backs off. On accept, the supervisor clamps `heartbeat_interval_secs` to a minimum of 5 seconds. +4. **Main select loop**: Concurrently reads inbound `GatewayMessage`s and fires heartbeat ticks. Inbound `Heartbeat` messages are acknowledged by the supervisor's outbound heartbeat cadence; `RelayOpen` and `RelayClose` are dispatched to `handle_gateway_message()`. +5. **Reconnect**: Any error in the session (stream error, connect failure, rejected hello) is reported as an OCSF event and the loop sleeps with exponential backoff (`INITIAL_BACKOFF = 1s`, doubled up to `MAX_BACKOFF = 30s`) before redialing. + +### Relay bridge loop + +`handle_gateway_message()` is a synchronous dispatcher. When a `RelayOpen { channel_id }` arrives, it spawns a dedicated task running `handle_relay_open()`. That task: + +1. Creates an outbound `mpsc::channel::(16)` wrapped in a `ReceiverStream`. +2. Sends `RelayFrame { payload: RelayInit { channel_id } }` as the first frame -- this claims the matching pending-relay slot on the gateway. +3. Calls `OpenShellClient::relay_stream(outbound)` on the shared `Channel`. This opens a new HTTP/2 stream on the existing connection -- no new TCP or TLS handshake. +4. `UnixStream::connect(ssh_socket_path)` dials the local sshd. The split read/write halves become the local endpoints of the bridge. +5. Spawns a task that reads from the Unix socket in 16 KiB chunks (`RELAY_CHUNK_SIZE`, matching the default HTTP/2 frame size) and forwards each chunk as `RelayFrame::Data` on the outbound stream. +6. The main loop drains inbound `RelayFrame::Data` messages and writes them to the socket. Non-data inbound frames (e.g. a second `Init`) are treated as protocol errors. +7. On any side closing, the bridge calls `ssh_w.shutdown()` to propagate EOF, drops the outbound sender to close the gRPC stream, and joins the reader task. + +The supervisor has no SSH or HTTP awareness -- it is purely a byte bridge. The protocol on top of the relay is whatever the gateway's caller (interactive `sandbox connect`, `ExecSandbox`, `rsync`-over-ssh) speaks to the sshd. + +### Session supersede + +If the gateway restarts or the sandbox restarts and reconnects with a new `instance_id` for the same `sandbox_id`, the gateway atomically replaces any prior session it has recorded. The new supervisor continues normally; the old stream (if still live on the gateway side) is torn down by the gateway's `remove_if_current` logic. Supervisors never need to coordinate between themselves -- each just keeps trying to connect, and the most recent `Hello` wins. + +If the gateway closes the stream cleanly (`inbound.message()` returns `Ok(None)`), `run_single_session` returns `Ok(())` and a `session_closed` event is emitted. Otherwise the loop reconnects. + +### OCSF telemetry + +Every session and relay transition emits an OCSF `NetworkActivity` event via `ocsf_emit!()` so operators can audit the control-plane connection from the sandbox's own logs. All events are built in `supervisor_session.rs` and covered by unit tests in the `ocsf_event_tests` module. + +| Helper | `activity_id` | `severity` | `status` | Fires when | +|--------|---------------|------------|----------|------------| +| `session_established_event` | `Open` | `Informational` | `Success` | After `SessionAccepted`, includes `session_id` and `heartbeat_secs` in the message | +| `session_closed_event` | `Close` | `Informational` | `Success` | Gateway closed the stream cleanly (`Ok(None)`) | +| `session_failed_event` | `Fail` | `Low` | `Failure` | Connect failed, hello rejected, or stream errored. Includes reconnect attempt counter | +| `relay_open_event` | `Open` | `Informational` | `Success` | `RelayOpen` received from the gateway | +| `relay_closed_event` | `Close` | `Informational` | `Success` | Relay bridge task exited without error | +| `relay_failed_event` | `Fail` | `Low` | `Failure` | Bridge task returned an error (e.g., socket write failure, inbound non-data frame) | +| `relay_close_from_gateway_event` | `Close` | `Informational` | -- | Gateway sent an explicit `RelayClose` on the control stream, with its `reason` | + +The `dst_endpoint` on session events is parsed from the gateway URI by `ocsf_gateway_endpoint()`. Relay events omit a destination (the bridge is sandbox-internal). + ## Zombie Reaping (PID 1 Init Duties) `openshell-sandbox` runs as PID 1 inside the container. In Linux, when a process exits, its parent must call `waitpid()` to collect the exit status; otherwise the process remains as a zombie. Orphaned processes (whose parent exits first) are reparented to PID 1, which becomes responsible for reaping them. @@ -1395,9 +1479,7 @@ This two-phase approach (peek with `WNOWAIT`, then selectively reap) avoids `ECH | `OPENSHELL_LOG_LEVEL` | `--log-level` | `warn` | Log level (trace/debug/info/warn/error) | | `OPENSHELL_POLICY_POLL_INTERVAL_SECS` | | `30` | Poll interval for gRPC policy updates (seconds). Only active in gRPC mode. | | `OPENSHELL_LOG_PUSH_LEVEL` | | `info` | Maximum tracing level for log push to gateway. Events above this level are not streamed. Only active in gRPC mode. | -| `OPENSHELL_SSH_LISTEN_ADDR` | `--ssh-listen-addr` | | SSH server bind address | -| `OPENSHELL_SSH_HANDSHAKE_SECRET` | `--ssh-handshake-secret` | | HMAC secret for SSH handshake | -| `OPENSHELL_SSH_HANDSHAKE_SKEW_SECS` | `--ssh-handshake-skew-secs` | `300` | Allowed clock skew for handshake | +| `OPENSHELL_SSH_SOCKET_PATH` | `--ssh-socket-path` | | Filesystem path to the Unix socket the embedded sshd binds (e.g. `/run/openshell/ssh.sock`). | | `OPENSHELL_INFERENCE_ROUTES` | `--inference-routes` | | Path to YAML inference routes file for standalone routing | ### Injected into child process @@ -1474,7 +1556,15 @@ The sandbox uses `miette` for error reporting and `thiserror` for typed errors. | Credential injection: path credential contains traversal/separator | HTTP 500, connection closed (fail-closed) | | Credential injection: percent-encoded placeholder bypass attempt | HTTP 500, connection closed (fail-closed) | | L7 parse error | Close the connection | -| SSH server failure | Async task error logged, main process unaffected | +| SSH socket bind failure | Fatal -- reported through the readiness channel and aborts startup | +| SSH server accept failure | Async task error logged, main process unaffected | +| Supervisor session: connect failure | Emit `session_failed` OCSF event, sleep with exponential backoff (1s -> 30s) and reconnect | +| Supervisor session: `SessionRejected` | Emit `session_failed` event with rejection reason; backoff and reconnect | +| Supervisor session: stream error mid-session | Emit `session_failed` event; backoff and reconnect | +| Supervisor session: gateway closes stream cleanly | Emit `session_closed` event and exit the task (no reconnect) | +| Relay bridge: `RelayStream` RPC failure | Emit `relay_failed` event; the individual relay is abandoned, the session stays up | +| Relay bridge: Unix socket connect failure | Emit `relay_failed` event; gateway observes EOF on the RelayStream | +| Relay bridge: non-data inbound frame after Init | Emit `relay_failed` event with protocol error | | Process timeout | Kill process, return exit code 124 | ## Logging @@ -1487,6 +1577,7 @@ Key structured log events: - `CONNECT`: One per proxy CONNECT request (for non-`inference.local` targets) with full identity context. Inference interception failures produce a separate `info!()` log with `action=deny` and the denial reason. - `BYPASS_DETECT`: One per detected direct connection attempt that bypassed the HTTP CONNECT proxy. Includes destination, protocol, process identity (best-effort), and remediation hint. Emitted at `warn` level. - `L7_REQUEST`: One per L7-inspected request with method, path, and decision +- Supervisor session / relay OCSF events: `session_established`, `session_closed`, `session_failed`, `relay_open`, `relay_closed`, `relay_failed`, `relay_close_from_gateway` (see [Supervisor Session](#supervisor-session)). - Sandbox lifecycle events: process start, exit, namespace creation/cleanup, bypass rule installation - Policy reload events: new version detected, reload success/failure, status report outcomes diff --git a/architecture/system-architecture.md b/architecture/system-architecture.md index 5ea92064ed..5c7fcdcf7c 100644 --- a/architecture/system-architecture.md +++ b/architecture/system-architecture.md @@ -28,8 +28,7 @@ graph TB subgraph GatewayPod["Gateway StatefulSet"] Gateway["openshell-server
:8080
(gRPC + HTTP, mTLS)"] SQLite[("SQLite DB
/var/openshell/
openshell.db")] - SandboxWatcher["Sandbox Watcher"] - KubeEventTailer["Kube Event Tailer"] + SupRegistry["SupervisorSessionRegistry
(live sessions + pending relays)"] WatchBus["SandboxWatchBus
(in-memory broadcast)"] LogBus["TracingLogBus
(in-memory broadcast)"] end @@ -37,7 +36,8 @@ graph TB subgraph SandboxPod["Sandbox Pod (1 per sandbox)"] subgraph Supervisor["Sandbox Supervisor
(privileged user)"] - SSHServer["Embedded SSH
Server (russh)
:2222"] + SSHServer["Embedded SSH
Server (russh)
Unix socket
/run/openshell/ssh.sock"] + RelayBridge["Relay Bridge
(ConnectSupervisor +
RelayStream client)"] Proxy["HTTP CONNECT
Proxy
10.200.0.1:3128"] OPA["OPA Policy Engine
(regorus, in-process)"] InferenceRouter["Inference Router
(openshell-router)"] @@ -101,10 +101,16 @@ graph TB %% CONNECTIONS: Gateway internals %% ============================================================ Gateway --> SQLite + Gateway --> SupRegistry Gateway -- "Watch + CRUD
Sandbox CRDs" --> KubeAPI - SandboxWatcher -- "status changes" --> WatchBus - KubeEventTailer -- "K8s events" --> Gateway - Gateway -- "NSSH1 handshake
(HMAC-SHA256) + SSH
:2222" --> SSHServer + KubeAPI -- "compute-driver events
(status, platform events)" --> Gateway + + %% ============================================================ + %% CONNECTIONS: Supervisor session (inbound from sandbox) + %% ============================================================ + RelayBridge -- "ConnectSupervisor
(persistent bidi stream)" --> SupRegistry + RelayBridge -- "RelayStream
(per-invocation byte bridge,
same HTTP/2 connection)" --> SupRegistry + RelayBridge -- "Unix socket
SSH bytes" --> SSHServer %% ============================================================ %% CONNECTIONS: CRD Controller @@ -123,7 +129,7 @@ graph TB %% ============================================================ %% CONNECTIONS: Sandbox --> Gateway (control plane) %% ============================================================ - Supervisor -- "gRPC (mTLS):
GetSandboxSettings
(policy + settings),
GetProviderEnvironment,
GetInferenceBundle,
PushSandboxLogs" --> Gateway + Supervisor -- "gRPC (mTLS):
GetSandboxConfig
(policy + settings),
GetProviderEnvironment,
GetInferenceBundle,
PushSandboxLogs" --> Gateway %% ============================================================ %% CONNECTIONS: Sandbox --> External (via proxy) @@ -145,9 +151,9 @@ graph TB K3s -- "pulls images
at runtime" --> GHCR %% ============================================================ - %% FILE SYNC + %% CLIENT SSH / EXEC (bytes tunneled via supervisor relay) %% ============================================================ - CLI -- "tar-over-SSH
(file sync)" --> SSHServer + CLI -- "HTTP CONNECT /connect/ssh
+ tar-over-SSH file sync
(bytes bridged through
SupervisorSessionRegistry)" --> Gateway %% ============================================================ %% STYLES @@ -164,8 +170,8 @@ graph TB classDef config fill:#90A4AE,stroke:#607D8B,color:#fff class CLI,TUI,SDK userComponent - class Gateway,SandboxWatcher,KubeEventTailer,WatchBus,LogBus gateway - class SSHServer,Proxy,OPA,InferenceRouter,CertCache sandbox + class Gateway,SupRegistry,WatchBus,LogBus gateway + class SSHServer,RelayBridge,Proxy,OPA,InferenceRouter,CertCache sandbox class Agent,Landlock,Seccomp,NetNS agent class SQLite datastore class Anthropic,OpenAI,NVIDIA_API,GitHub,GitLab,PyPI,NPM,LMStudio,VLLM,GHCR external @@ -189,12 +195,14 @@ graph TB 1. **CLI/SDK to Gateway**: All control-plane traffic uses gRPC over HTTPS with mutual TLS (mTLS). Single multiplexed port (8080 inside cluster, 30051 NodePort). -2. **SSH Access**: CLI connects via HTTP CONNECT upgrade at `/connect/ssh`. Gateway authenticates with session token, then bridges to sandbox SSH (port 2222) using NSSH1 HMAC-SHA256 handshake. +2. **Supervisor Session (inbound from sandbox)**: Each sandbox supervisor opens a persistent `ConnectSupervisor` bidi gRPC stream to the gateway over mTLS. The gateway tracks these in `SupervisorSessionRegistry`. When SSH or exec access is needed, the gateway sends `RelayOpen { channel_id }` on that stream; the supervisor responds by initiating a `RelayStream` RPC on the same HTTP/2 connection whose first frame is a `RelayInit { channel_id }`. Subsequent frames carry raw bytes in both directions. The gateway never dials the sandbox pod. + +3. **SSH / Exec Access**: CLI connects via HTTP CONNECT upgrade at `/connect/ssh` (or calls `ExecSandbox` gRPC). The gateway authenticates, calls `open_relay`, and bridges the client bytes through the supervisor's `RelayStream` to the supervisor's in-sandbox SSH daemon, which binds to a Unix socket (`/run/openshell/ssh.sock`) rather than a TCP port. -3. **File Sync**: tar archives streamed over the SSH tunnel (no rsync dependency). +4. **File Sync**: tar archives streamed over the relay-tunneled SSH session (no rsync dependency). -4. **Sandbox to External**: All agent outbound traffic is forced through the HTTP CONNECT proxy (10.200.0.1:3128) via a network namespace veth pair. OPA/Rego policies evaluate every connection. TLS is automatically detected and terminated for credential injection; endpoints with `protocol` configured also get L7 request-level inspection. +5. **Sandbox to External**: All agent outbound traffic is forced through the HTTP CONNECT proxy (10.200.0.1:3128) via a network namespace veth pair. OPA/Rego policies evaluate every connection. TLS is automatically detected and terminated for credential injection; endpoints with `protocol` configured also get L7 request-level inspection. -5. **Inference Routing**: Inference requests are handled inside the sandbox by the openshell-router (not through the gateway). The gateway provides route configuration and credentials via gRPC; the sandbox executes HTTP requests directly to inference backends. +6. **Inference Routing**: Inference requests are handled inside the sandbox by the openshell-router (not through the gateway). The gateway provides route configuration and credentials via gRPC; the sandbox executes HTTP requests directly to inference backends. -6. **Sandbox to Gateway**: The sandbox supervisor uses gRPC (mTLS) to fetch policies and runtime settings (via `GetSandboxSettings`), provider credentials, inference bundles, and to push logs back to the gateway. The settings channel delivers typed key-value pairs alongside policy through a unified poll loop. +7. **Sandbox to Gateway (control plane)**: The sandbox supervisor uses gRPC (mTLS) to fetch policies and runtime settings (via `GetSandboxConfig`), provider credentials, inference bundles, and to push logs back to the gateway. The settings channel delivers typed key-value pairs alongside policy through a unified poll loop. This reuses the same mTLS connection that carries `ConnectSupervisor`. diff --git a/crates/openshell-cli/src/ssh.rs b/crates/openshell-cli/src/ssh.rs index 2bee1b7bc0..0f7e8ee0b0 100644 --- a/crates/openshell-cli/src/ssh.rs +++ b/crates/openshell-cli/src/ssh.rs @@ -145,7 +145,15 @@ fn ssh_base_command(proxy_command: &str) -> Command { .arg("-o") .arg("GlobalKnownHostsFile=/dev/null") .arg("-o") - .arg("LogLevel=ERROR"); + .arg("LogLevel=ERROR") + // Detect a dead relay within ~45s. The relay rides on a TCP connection + // that the client has no way to observe silently dropping (gateway + // restart, supervisor restart, cluster failover), so fall back to + // SSH-level keepalives instead of hanging forever. + .arg("-o") + .arg("ServerAliveInterval=15") + .arg("-o") + .arg("ServerAliveCountMax=3"); command } @@ -877,7 +885,7 @@ fn render_ssh_config(gateway: &str, name: &str) -> String { ); let host_alias = host_alias(name); format!( - "Host {host_alias}\n User sandbox\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n GlobalKnownHostsFile /dev/null\n LogLevel ERROR\n ProxyCommand {proxy_cmd}\n" + "Host {host_alias}\n User sandbox\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n GlobalKnownHostsFile /dev/null\n LogLevel ERROR\n ServerAliveInterval 15\n ServerAliveCountMax 3\n ProxyCommand {proxy_cmd}\n" ) } diff --git a/crates/openshell-cli/tests/ensure_providers_integration.rs b/crates/openshell-cli/tests/ensure_providers_integration.rs index 2cd3620239..a3dd6826fe 100644 --- a/crates/openshell-cli/tests/ensure_providers_integration.rs +++ b/crates/openshell-cli/tests/ensure_providers_integration.rs @@ -11,13 +11,14 @@ use openshell_core::proto::open_shell_server::{OpenShell, OpenShellServer}; use openshell_core::proto::{ CreateProviderRequest, CreateSandboxRequest, CreateSshSessionRequest, CreateSshSessionResponse, DeleteProviderRequest, DeleteProviderResponse, DeleteSandboxRequest, DeleteSandboxResponse, - ExecSandboxEvent, ExecSandboxRequest, GetGatewayConfigRequest, GetGatewayConfigResponse, - GetProviderRequest, GetSandboxConfigRequest, GetSandboxConfigResponse, - GetSandboxProviderEnvironmentRequest, GetSandboxProviderEnvironmentResponse, GetSandboxRequest, - HealthRequest, HealthResponse, ListProvidersRequest, ListProvidersResponse, - ListSandboxesRequest, ListSandboxesResponse, Provider, ProviderResponse, - RevokeSshSessionRequest, RevokeSshSessionResponse, SandboxResponse, SandboxStreamEvent, - ServiceStatus, UpdateProviderRequest, WatchSandboxRequest, + ExecSandboxEvent, ExecSandboxRequest, GatewayMessage, GetGatewayConfigRequest, + GetGatewayConfigResponse, GetProviderRequest, GetSandboxConfigRequest, + GetSandboxConfigResponse, GetSandboxProviderEnvironmentRequest, + GetSandboxProviderEnvironmentResponse, GetSandboxRequest, HealthRequest, HealthResponse, + ListProvidersRequest, ListProvidersResponse, ListSandboxesRequest, ListSandboxesResponse, + Provider, ProviderResponse, RevokeSshSessionRequest, RevokeSshSessionResponse, SandboxResponse, + SandboxStreamEvent, ServiceStatus, SupervisorMessage, UpdateProviderRequest, + WatchSandboxRequest, }; use rcgen::{ BasicConstraints, Certificate, CertificateParams, ExtendedKeyUsagePurpose, IsCa, KeyPair, @@ -298,6 +299,8 @@ impl OpenShell for TestOpenShell { tokio_stream::wrappers::ReceiverStream>; type ExecSandboxStream = tokio_stream::wrappers::ReceiverStream>; + type ConnectSupervisorStream = + tokio_stream::wrappers::ReceiverStream>; async fn watch_sandbox( &self, @@ -423,6 +426,24 @@ impl OpenShell for TestOpenShell { ) -> Result, Status> { Err(Status::unimplemented("not implemented in test")) } + + async fn connect_supervisor( + &self, + _request: tonic::Request>, + ) -> Result, Status> { + Err(Status::unimplemented("not implemented in test")) + } + + type RelayStreamStream = tokio_stream::wrappers::ReceiverStream< + Result, + >; + + async fn relay_stream( + &self, + _request: tonic::Request>, + ) -> Result, tonic::Status> { + Err(tonic::Status::unimplemented("not implemented in test")) + } } // ── TLS helpers ────────────────────────────────────────────────────── diff --git a/crates/openshell-cli/tests/mtls_integration.rs b/crates/openshell-cli/tests/mtls_integration.rs index 5d04239bf5..e78c915782 100644 --- a/crates/openshell-cli/tests/mtls_integration.rs +++ b/crates/openshell-cli/tests/mtls_integration.rs @@ -200,6 +200,9 @@ impl OpenShell for TestOpenShell { >; type ExecSandboxStream = tokio_stream::wrappers::ReceiverStream>; + type ConnectSupervisorStream = tokio_stream::wrappers::ReceiverStream< + Result, + >; async fn watch_sandbox( &self, @@ -325,6 +328,24 @@ impl OpenShell for TestOpenShell { ) -> Result, Status> { Err(Status::unimplemented("not implemented in test")) } + + async fn connect_supervisor( + &self, + _request: tonic::Request>, + ) -> Result, Status> { + Err(Status::unimplemented("not implemented in test")) + } + + type RelayStreamStream = tokio_stream::wrappers::ReceiverStream< + Result, + >; + + async fn relay_stream( + &self, + _request: tonic::Request>, + ) -> Result, tonic::Status> { + Err(tonic::Status::unimplemented("not implemented in test")) + } } fn build_ca() -> (Certificate, KeyPair) { diff --git a/crates/openshell-cli/tests/provider_commands_integration.rs b/crates/openshell-cli/tests/provider_commands_integration.rs index c5476afee7..dc6ec9d4cb 100644 --- a/crates/openshell-cli/tests/provider_commands_integration.rs +++ b/crates/openshell-cli/tests/provider_commands_integration.rs @@ -7,13 +7,14 @@ use openshell_core::proto::open_shell_server::{OpenShell, OpenShellServer}; use openshell_core::proto::{ CreateProviderRequest, CreateSandboxRequest, CreateSshSessionRequest, CreateSshSessionResponse, DeleteProviderRequest, DeleteProviderResponse, DeleteSandboxRequest, DeleteSandboxResponse, - ExecSandboxEvent, ExecSandboxRequest, GetGatewayConfigRequest, GetGatewayConfigResponse, - GetProviderRequest, GetSandboxConfigRequest, GetSandboxConfigResponse, - GetSandboxProviderEnvironmentRequest, GetSandboxProviderEnvironmentResponse, GetSandboxRequest, - HealthRequest, HealthResponse, ListProvidersRequest, ListProvidersResponse, - ListSandboxesRequest, ListSandboxesResponse, Provider, ProviderResponse, - RevokeSshSessionRequest, RevokeSshSessionResponse, SandboxResponse, SandboxStreamEvent, - ServiceStatus, UpdateProviderRequest, WatchSandboxRequest, + ExecSandboxEvent, ExecSandboxRequest, GatewayMessage, GetGatewayConfigRequest, + GetGatewayConfigResponse, GetProviderRequest, GetSandboxConfigRequest, + GetSandboxConfigResponse, GetSandboxProviderEnvironmentRequest, + GetSandboxProviderEnvironmentResponse, GetSandboxRequest, HealthRequest, HealthResponse, + ListProvidersRequest, ListProvidersResponse, ListSandboxesRequest, ListSandboxesResponse, + Provider, ProviderResponse, RevokeSshSessionRequest, RevokeSshSessionResponse, SandboxResponse, + SandboxStreamEvent, ServiceStatus, SupervisorMessage, UpdateProviderRequest, + WatchSandboxRequest, }; use rcgen::{ BasicConstraints, Certificate, CertificateParams, ExtendedKeyUsagePurpose, IsCa, KeyPair, @@ -252,6 +253,8 @@ impl OpenShell for TestOpenShell { tokio_stream::wrappers::ReceiverStream>; type ExecSandboxStream = tokio_stream::wrappers::ReceiverStream>; + type ConnectSupervisorStream = + tokio_stream::wrappers::ReceiverStream>; async fn watch_sandbox( &self, @@ -377,6 +380,24 @@ impl OpenShell for TestOpenShell { ) -> Result, Status> { Err(Status::unimplemented("not implemented in test")) } + + async fn connect_supervisor( + &self, + _request: tonic::Request>, + ) -> Result, Status> { + Err(Status::unimplemented("not implemented in test")) + } + + type RelayStreamStream = tokio_stream::wrappers::ReceiverStream< + Result, + >; + + async fn relay_stream( + &self, + _request: tonic::Request>, + ) -> Result, tonic::Status> { + Err(tonic::Status::unimplemented("not implemented in test")) + } } fn install_rustls_provider() { diff --git a/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs b/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs index d5d39f0821..50a4fa6514 100644 --- a/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs +++ b/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs @@ -8,14 +8,14 @@ use openshell_core::proto::open_shell_server::{OpenShell, OpenShellServer}; use openshell_core::proto::{ CreateProviderRequest, CreateSandboxRequest, CreateSshSessionRequest, CreateSshSessionResponse, DeleteProviderRequest, DeleteProviderResponse, DeleteSandboxRequest, DeleteSandboxResponse, - ExecSandboxEvent, ExecSandboxRequest, GetGatewayConfigRequest, GetGatewayConfigResponse, - GetProviderRequest, GetSandboxConfigRequest, GetSandboxConfigResponse, - GetSandboxProviderEnvironmentRequest, GetSandboxProviderEnvironmentResponse, GetSandboxRequest, - HealthRequest, HealthResponse, ListProvidersRequest, ListProvidersResponse, - ListSandboxesRequest, ListSandboxesResponse, PlatformEvent, ProviderResponse, - RevokeSshSessionRequest, RevokeSshSessionResponse, Sandbox, SandboxPhase, SandboxResponse, - SandboxStreamEvent, ServiceStatus, UpdateProviderRequest, WatchSandboxRequest, - sandbox_stream_event, + ExecSandboxEvent, ExecSandboxRequest, GatewayMessage, GetGatewayConfigRequest, + GetGatewayConfigResponse, GetProviderRequest, GetSandboxConfigRequest, + GetSandboxConfigResponse, GetSandboxProviderEnvironmentRequest, + GetSandboxProviderEnvironmentResponse, GetSandboxRequest, HealthRequest, HealthResponse, + ListProvidersRequest, ListProvidersResponse, ListSandboxesRequest, ListSandboxesResponse, + PlatformEvent, ProviderResponse, RevokeSshSessionRequest, RevokeSshSessionResponse, Sandbox, + SandboxPhase, SandboxResponse, SandboxStreamEvent, ServiceStatus, SupervisorMessage, + UpdateProviderRequest, WatchSandboxRequest, sandbox_stream_event, }; use rcgen::{ BasicConstraints, Certificate, CertificateParams, ExtendedKeyUsagePurpose, IsCa, KeyPair, @@ -242,6 +242,8 @@ impl OpenShell for TestOpenShell { tokio_stream::wrappers::ReceiverStream>; type ExecSandboxStream = tokio_stream::wrappers::ReceiverStream>; + type ConnectSupervisorStream = + tokio_stream::wrappers::ReceiverStream>; async fn watch_sandbox( &self, @@ -403,6 +405,24 @@ impl OpenShell for TestOpenShell { ) -> Result, Status> { Err(Status::unimplemented("not implemented in test")) } + + async fn connect_supervisor( + &self, + _request: tonic::Request>, + ) -> Result, Status> { + Err(Status::unimplemented("not implemented in test")) + } + + type RelayStreamStream = tokio_stream::wrappers::ReceiverStream< + Result, + >; + + async fn relay_stream( + &self, + _request: tonic::Request>, + ) -> Result, tonic::Status> { + Err(tonic::Status::unimplemented("not implemented in test")) + } } fn install_rustls_provider() { diff --git a/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs b/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs index 885f659d00..d59356b53c 100644 --- a/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs +++ b/crates/openshell-cli/tests/sandbox_name_fallback_integration.rs @@ -8,12 +8,13 @@ use openshell_core::proto::open_shell_server::{OpenShell, OpenShellServer}; use openshell_core::proto::{ CreateProviderRequest, CreateSandboxRequest, CreateSshSessionRequest, CreateSshSessionResponse, DeleteProviderRequest, DeleteProviderResponse, DeleteSandboxRequest, DeleteSandboxResponse, - ExecSandboxEvent, ExecSandboxRequest, GetGatewayConfigRequest, GetGatewayConfigResponse, - GetProviderRequest, GetSandboxConfigRequest, GetSandboxConfigResponse, - GetSandboxProviderEnvironmentRequest, GetSandboxProviderEnvironmentResponse, GetSandboxRequest, - HealthRequest, HealthResponse, ListProvidersRequest, ListProvidersResponse, - ListSandboxesRequest, ListSandboxesResponse, ProviderResponse, Sandbox, SandboxPolicy, - SandboxResponse, SandboxStreamEvent, ServiceStatus, UpdateProviderRequest, WatchSandboxRequest, + ExecSandboxEvent, ExecSandboxRequest, GatewayMessage, GetGatewayConfigRequest, + GetGatewayConfigResponse, GetProviderRequest, GetSandboxConfigRequest, + GetSandboxConfigResponse, GetSandboxProviderEnvironmentRequest, + GetSandboxProviderEnvironmentResponse, GetSandboxRequest, HealthRequest, HealthResponse, + ListProvidersRequest, ListProvidersResponse, ListSandboxesRequest, ListSandboxesResponse, + ProviderResponse, Sandbox, SandboxPolicy, SandboxResponse, SandboxStreamEvent, ServiceStatus, + SupervisorMessage, UpdateProviderRequest, WatchSandboxRequest, }; use rcgen::{ BasicConstraints, Certificate, CertificateParams, ExtendedKeyUsagePurpose, IsCa, KeyPair, @@ -221,6 +222,8 @@ impl OpenShell for TestOpenShell { tokio_stream::wrappers::ReceiverStream>; type ExecSandboxStream = tokio_stream::wrappers::ReceiverStream>; + type ConnectSupervisorStream = + tokio_stream::wrappers::ReceiverStream>; async fn watch_sandbox( &self, @@ -346,6 +349,24 @@ impl OpenShell for TestOpenShell { ) -> Result, Status> { Err(Status::unimplemented("not implemented in test")) } + + async fn connect_supervisor( + &self, + _request: tonic::Request>, + ) -> Result, Status> { + Err(Status::unimplemented("not implemented in test")) + } + + type RelayStreamStream = tokio_stream::wrappers::ReceiverStream< + Result, + >; + + async fn relay_stream( + &self, + _request: tonic::Request>, + ) -> Result, tonic::Status> { + Err(tonic::Status::unimplemented("not implemented in test")) + } } // ── helpers ─────────────────────────────────────────────────────────── diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index 01b5c23720..7a9141c520 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -107,9 +107,17 @@ pub struct Config { #[serde(default = "default_ssh_connect_path")] pub ssh_connect_path: String, - /// SSH listen port inside sandbox pods. - #[serde(default = "default_sandbox_ssh_port")] - pub sandbox_ssh_port: u16, + /// Filesystem path where the sandbox supervisor binds its SSH Unix + /// socket. The supervisor is passed this path via + /// `OPENSHELL_SSH_SOCKET_PATH` / `--ssh-socket-path` and connects its + /// relay bridge to the same path. + /// + /// When the gateway orchestrates sandboxes that each live in their own + /// filesystem (K8s pod, libkrun VM, etc.), the default is safe. For + /// local dev where multiple supervisors share `/run`, override this to + /// something unique per sandbox. + #[serde(default = "default_sandbox_ssh_socket_path")] + pub sandbox_ssh_socket_path: String, /// Shared secret for gateway-to-sandbox SSH handshake. #[serde(default)] @@ -179,7 +187,7 @@ impl Config { ssh_gateway_host: default_ssh_gateway_host(), ssh_gateway_port: default_ssh_gateway_port(), ssh_connect_path: default_ssh_connect_path(), - sandbox_ssh_port: default_sandbox_ssh_port(), + sandbox_ssh_socket_path: default_sandbox_ssh_socket_path(), ssh_handshake_secret: String::new(), ssh_handshake_skew_secs: default_ssh_handshake_skew_secs(), ssh_session_ttl_secs: default_ssh_session_ttl_secs(), @@ -268,13 +276,6 @@ impl Config { self } - /// Create a new configuration with the sandbox SSH port. - #[must_use] - pub const fn with_sandbox_ssh_port(mut self, port: u16) -> Self { - self.sandbox_ssh_port = port; - self - } - /// Create a new configuration with the SSH handshake secret. #[must_use] pub fn with_ssh_handshake_secret(mut self, secret: impl Into) -> Self { @@ -339,8 +340,8 @@ fn default_ssh_connect_path() -> String { "/connect/ssh".to_string() } -const fn default_sandbox_ssh_port() -> u16 { - 2222 +fn default_sandbox_ssh_socket_path() -> String { + "/run/openshell/ssh.sock".to_string() } const fn default_ssh_handshake_skew_secs() -> u64 { diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 3ce98eae89..be36661307 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -7,8 +7,7 @@ pub struct KubernetesComputeConfig { pub default_image: String, pub image_pull_policy: String, pub grpc_endpoint: String, - pub ssh_listen_addr: String, - pub ssh_port: u16, + pub ssh_socket_path: String, pub ssh_handshake_secret: String, pub ssh_handshake_skew_secs: u64, pub client_tls_secret_name: String, diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 440703af57..f70054805b 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -5,7 +5,7 @@ use crate::config::KubernetesComputeConfig; use futures::{Stream, StreamExt, TryStreamExt}; -use k8s_openapi::api::core::v1::{Event as KubeEventObj, Node, Pod}; +use k8s_openapi::api::core::v1::{Event as KubeEventObj, Node}; use kube::api::{Api, ApiResource, DeleteParams, ListParams, PostParams}; use kube::core::gvk::GroupVersionKind; use kube::core::{DynamicObject, ObjectMeta}; @@ -15,12 +15,10 @@ use openshell_core::proto::compute::v1::{ DriverCondition as SandboxCondition, DriverPlatformEvent as PlatformEvent, DriverSandbox as Sandbox, DriverSandboxSpec as SandboxSpec, DriverSandboxStatus as SandboxStatus, DriverSandboxTemplate as SandboxTemplate, - GetCapabilitiesResponse, ResolveSandboxEndpointResponse, SandboxEndpoint, - WatchSandboxesDeletedEvent, WatchSandboxesEvent, WatchSandboxesPlatformEvent, - WatchSandboxesSandboxEvent, sandbox_endpoint, watch_sandboxes_event, + GetCapabilitiesResponse, WatchSandboxesDeletedEvent, WatchSandboxesEvent, + WatchSandboxesPlatformEvent, WatchSandboxesSandboxEvent, watch_sandboxes_event, }; use std::collections::BTreeMap; -use std::net::IpAddr; use std::pin::Pin; use std::time::Duration; use tokio::sync::mpsc; @@ -149,8 +147,8 @@ impl KubernetesComputeDriver { &self.config.namespace } - pub fn ssh_listen_addr(&self) -> &str { - &self.config.ssh_listen_addr + pub fn ssh_socket_path(&self) -> &str { + &self.config.ssh_socket_path } pub const fn ssh_handshake_skew_secs(&self) -> u64 { @@ -271,21 +269,6 @@ impl KubernetesComputeDriver { &self.config.ssh_handshake_secret } - async fn agent_pod_ip(&self, pod_name: &str) -> Result, KubeError> { - let api: Api = Api::namespaced(self.client.clone(), &self.config.namespace); - match api.get(pod_name).await { - Ok(pod) => { - let ip = pod - .status - .and_then(|status| status.pod_ip) - .and_then(|ip| ip.parse().ok()); - Ok(ip) - } - Err(KubeError::Api(err)) if err.code == 404 => Ok(None), - Err(err) => Err(err), - } - } - pub async fn create_sandbox(&self, sandbox: &Sandbox) -> Result<(), KubernetesDriverError> { let name = sandbox.name.as_str(); info!( @@ -311,7 +294,7 @@ impl KubernetesComputeDriver { &sandbox.id, &sandbox.name, &self.config.grpc_endpoint, - self.ssh_listen_addr(), + self.ssh_socket_path(), self.ssh_handshake_secret(), self.ssh_handshake_skew_secs(), &self.config.client_tls_secret_name, @@ -407,52 +390,6 @@ impl KubernetesComputeDriver { } } - pub async fn resolve_sandbox_endpoint( - &self, - sandbox: &Sandbox, - ) -> Result { - if let Some(status) = sandbox.status.as_ref() - && !status.instance_id.is_empty() - { - match self.agent_pod_ip(&status.instance_id).await { - Ok(Some(ip)) => { - return Ok(ResolveSandboxEndpointResponse { - endpoint: Some(SandboxEndpoint { - target: Some(sandbox_endpoint::Target::Ip(ip.to_string())), - port: u32::from(self.config.ssh_port), - }), - }); - } - Ok(None) => { - return Err(KubernetesDriverError::Precondition( - "sandbox agent pod IP is not available".to_string(), - )); - } - Err(err) => { - return Err(KubernetesDriverError::Message(format!( - "failed to resolve agent pod IP: {err}" - ))); - } - } - } - - if sandbox.name.is_empty() { - return Err(KubernetesDriverError::Precondition( - "sandbox has no name".to_string(), - )); - } - - Ok(ResolveSandboxEndpointResponse { - endpoint: Some(SandboxEndpoint { - target: Some(sandbox_endpoint::Target::Host(format!( - "{}.{}.svc.cluster.local", - sandbox.name, self.config.namespace - ))), - port: u32::from(self.config.ssh_port), - }), - }) - } - pub async fn watch_sandboxes(&self) -> Result { let namespace = self.config.namespace.clone(); let sandbox_api = self.api(); @@ -925,7 +862,7 @@ fn sandbox_to_k8s_spec( sandbox_id: &str, sandbox_name: &str, grpc_endpoint: &str, - ssh_listen_addr: &str, + ssh_socket_path: &str, ssh_handshake_secret: &str, ssh_handshake_skew_secs: u64, client_tls_secret_name: &str, @@ -965,7 +902,7 @@ fn sandbox_to_k8s_spec( sandbox_id, sandbox_name, grpc_endpoint, - ssh_listen_addr, + ssh_socket_path, ssh_handshake_secret, ssh_handshake_skew_secs, &spec.environment, @@ -1011,7 +948,7 @@ fn sandbox_to_k8s_spec( sandbox_id, sandbox_name, grpc_endpoint, - ssh_listen_addr, + ssh_socket_path, ssh_handshake_secret, ssh_handshake_skew_secs, spec_env, @@ -1036,7 +973,7 @@ fn sandbox_template_to_k8s( sandbox_id: &str, sandbox_name: &str, grpc_endpoint: &str, - ssh_listen_addr: &str, + ssh_socket_path: &str, ssh_handshake_secret: &str, ssh_handshake_skew_secs: u64, spec_environment: &std::collections::HashMap, @@ -1089,7 +1026,7 @@ fn sandbox_template_to_k8s( sandbox_id, sandbox_name, grpc_endpoint, - ssh_listen_addr, + ssh_socket_path, ssh_handshake_secret, ssh_handshake_skew_secs, !client_tls_secret_name.is_empty(), @@ -1239,7 +1176,7 @@ fn build_env_list( sandbox_id: &str, sandbox_name: &str, grpc_endpoint: &str, - ssh_listen_addr: &str, + ssh_socket_path: &str, ssh_handshake_secret: &str, ssh_handshake_skew_secs: u64, tls_enabled: bool, @@ -1252,7 +1189,7 @@ fn build_env_list( sandbox_id, sandbox_name, grpc_endpoint, - ssh_listen_addr, + ssh_socket_path, ssh_handshake_secret, ssh_handshake_skew_secs, tls_enabled, @@ -1274,7 +1211,7 @@ fn apply_required_env( sandbox_id: &str, sandbox_name: &str, grpc_endpoint: &str, - ssh_listen_addr: &str, + ssh_socket_path: &str, ssh_handshake_secret: &str, ssh_handshake_skew_secs: u64, tls_enabled: bool, @@ -1283,8 +1220,8 @@ fn apply_required_env( upsert_env(env, "OPENSHELL_SANDBOX", sandbox_name); upsert_env(env, "OPENSHELL_ENDPOINT", grpc_endpoint); upsert_env(env, "OPENSHELL_SANDBOX_COMMAND", "sleep infinity"); - if !ssh_listen_addr.is_empty() { - upsert_env(env, "OPENSHELL_SSH_LISTEN_ADDR", ssh_listen_addr); + if !ssh_socket_path.is_empty() { + upsert_env(env, "OPENSHELL_SSH_SOCKET_PATH", ssh_socket_path); } upsert_env(env, "OPENSHELL_SSH_HANDSHAKE_SECRET", ssh_handshake_secret); upsert_env( diff --git a/crates/openshell-driver-kubernetes/src/grpc.rs b/crates/openshell-driver-kubernetes/src/grpc.rs index 2c5a94467b..75e131d417 100644 --- a/crates/openshell-driver-kubernetes/src/grpc.rs +++ b/crates/openshell-driver-kubernetes/src/grpc.rs @@ -5,8 +5,7 @@ use futures::{Stream, StreamExt}; use openshell_core::proto::compute::v1::{ CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, GetCapabilitiesRequest, GetCapabilitiesResponse, GetSandboxRequest, GetSandboxResponse, - ListSandboxesRequest, ListSandboxesResponse, ResolveSandboxEndpointRequest, - ResolveSandboxEndpointResponse, StopSandboxRequest, StopSandboxResponse, + ListSandboxesRequest, ListSandboxesResponse, StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesEvent, WatchSandboxesRequest, compute_driver_server::ComputeDriver, }; @@ -128,21 +127,6 @@ impl ComputeDriver for ComputeDriverService { Ok(Response::new(DeleteSandboxResponse { deleted })) } - async fn resolve_sandbox_endpoint( - &self, - request: Request, - ) -> Result, Status> { - let sandbox = request - .into_inner() - .sandbox - .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; - self.driver - .resolve_sandbox_endpoint(&sandbox) - .await - .map(Response::new) - .map_err(status_from_driver_error) - } - type WatchSandboxesStream = Pin> + Send + 'static>>; diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 76c567f593..4b871d77f5 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -39,8 +39,12 @@ struct Args { #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] grpc_endpoint: Option, - #[arg(long, env = "OPENSHELL_SANDBOX_SSH_PORT", default_value_t = 2222)] - sandbox_ssh_port: u16, + #[arg( + long, + env = "OPENSHELL_SANDBOX_SSH_SOCKET_PATH", + default_value = "/run/openshell/ssh.sock" + )] + sandbox_ssh_socket_path: String, #[arg(long, env = "OPENSHELL_SSH_HANDSHAKE_SECRET")] ssh_handshake_secret: String, @@ -69,8 +73,7 @@ async fn main() -> Result<()> { default_image: args.sandbox_image.unwrap_or_default(), image_pull_policy: args.sandbox_image_pull_policy.unwrap_or_default(), grpc_endpoint: args.grpc_endpoint.unwrap_or_default(), - ssh_listen_addr: format!("0.0.0.0:{}", args.sandbox_ssh_port), - ssh_port: args.sandbox_ssh_port, + ssh_socket_path: args.sandbox_ssh_socket_path, ssh_handshake_secret: args.ssh_handshake_secret, ssh_handshake_skew_secs: args.ssh_handshake_skew_secs, client_tls_secret_name: args.client_tls_secret_name.unwrap_or_default(), diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 3d3fbf4b61..8237ba03c1 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -1,10 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use crate::{ - GUEST_SSH_PORT, - rootfs::{extract_sandbox_rootfs_to, sandbox_guest_init_path}, -}; +use crate::rootfs::{extract_sandbox_rootfs_to, sandbox_guest_init_path}; use futures::Stream; use nix::errno::Errno; use nix::sys::signal::{Signal, kill}; @@ -14,21 +11,18 @@ use openshell_core::proto::compute::v1::{ DriverCondition as SandboxCondition, DriverPlatformEvent as PlatformEvent, DriverSandbox as Sandbox, DriverSandboxStatus as SandboxStatus, GetCapabilitiesRequest, GetCapabilitiesResponse, GetSandboxRequest, GetSandboxResponse, ListSandboxesRequest, - ListSandboxesResponse, ResolveSandboxEndpointRequest, ResolveSandboxEndpointResponse, - SandboxEndpoint, StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, + ListSandboxesResponse, StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesDeletedEvent, WatchSandboxesEvent, WatchSandboxesPlatformEvent, WatchSandboxesRequest, WatchSandboxesSandboxEvent, - compute_driver_server::ComputeDriver, sandbox_endpoint, watch_sandboxes_event, + compute_driver_server::ComputeDriver, watch_sandboxes_event, }; use std::collections::{HashMap, HashSet}; -use std::net::{Ipv4Addr, SocketAddr, TcpListener}; use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; use std::pin::Pin; use std::process::Stdio; use std::sync::Arc; use std::time::Duration; -use tokio::net::TcpStream; use tokio::process::{Child, Command}; use tokio::sync::{Mutex, broadcast, mpsc}; use tokio_stream::wrappers::ReceiverStream; @@ -39,6 +33,7 @@ const DRIVER_NAME: &str = "openshell-driver-vm"; const WATCH_BUFFER: usize = 256; const DEFAULT_VCPUS: u8 = 2; const DEFAULT_MEM_MIB: u32 = 2048; +const GUEST_SSH_SOCKET_PATH: &str = "/run/openshell/ssh.sock"; const GUEST_TLS_DIR: &str = "/opt/openshell/tls"; const GUEST_TLS_CA_PATH: &str = "/opt/openshell/tls/ca.crt"; const GUEST_TLS_CERT_PATH: &str = "/opt/openshell/tls/tls.crt"; @@ -168,7 +163,6 @@ struct VmProcess { #[derive(Debug)] struct SandboxRecord { snapshot: Sandbox, - ssh_port: u16, state_dir: PathBuf, process: Arc>, } @@ -236,7 +230,6 @@ impl VmDriver { return Err(Status::already_exists("sandbox already exists")); } - let ssh_port = allocate_local_port()?; let state_dir = sandbox_state_dir(&self.config.state_dir, &sandbox.id); let rootfs = state_dir.join("rootfs"); @@ -279,9 +272,6 @@ impl VmDriver { .arg("--vm-krun-log-level") .arg(self.config.krun_log_level.to_string()); command.arg("--vm-console-output").arg(&console_output); - command - .arg("--vm-port") - .arg(format!("{ssh_port}:{GUEST_SSH_PORT}")); for env in build_guest_environment(sandbox, &self.config) { command.arg("--vm-env").arg(env); } @@ -308,7 +298,6 @@ impl VmDriver { sandbox.id.clone(), SandboxRecord { snapshot: snapshot.clone(), - ssh_port, state_dir: state_dir.clone(), process: process.clone(), }, @@ -385,25 +374,6 @@ impl VmDriver { Ok(DeleteSandboxResponse { deleted: true }) } - pub async fn resolve_endpoint( - &self, - sandbox: &Sandbox, - ) -> Result { - let registry = self.registry.lock().await; - let record = registry.get(&sandbox.id).or_else(|| { - registry - .values() - .find(|record| record.snapshot.name == sandbox.name) - }); - let record = record.ok_or_else(|| Status::not_found("sandbox not found"))?; - Ok(ResolveSandboxEndpointResponse { - endpoint: Some(SandboxEndpoint { - target: Some(sandbox_endpoint::Target::Host("127.0.0.1".to_string())), - port: u32::from(record.ssh_port), - }), - }) - } - pub async fn get_sandbox( &self, sandbox_id: &str, @@ -437,16 +407,12 @@ impl VmDriver { let mut ready_emitted = false; loop { - let (process, ssh_port, state_dir) = { + let (process, state_dir) = { let registry = self.registry.lock().await; let Some(record) = registry.get(&sandbox_id) else { return; }; - ( - record.process.clone(), - record.ssh_port, - record.state_dir.clone(), - ) + (record.process.clone(), record.state_dir.clone()) }; let exit_status = { @@ -503,8 +469,7 @@ impl VmDriver { return; } - if !ready_emitted && port_is_ready(ssh_port).await && guest_ssh_ready(&state_dir).await - { + if !ready_emitted && guest_ssh_ready(&state_dir).await { if let Some(snapshot) = self .set_snapshot_condition(&sandbox_id, ready_condition(), false) .await @@ -649,17 +614,6 @@ impl ComputeDriver for VmDriver { Ok(Response::new(response)) } - async fn resolve_sandbox_endpoint( - &self, - request: Request, - ) -> Result, Status> { - let sandbox = request - .into_inner() - .sandbox - .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; - Ok(Response::new(self.resolve_endpoint(&sandbox).await?)) - } - type WatchSandboxesStream = Pin> + Send + 'static>>; @@ -794,16 +748,8 @@ fn build_guest_environment(sandbox: &Sandbox, config: &VmDriverConfig) -> Vec Result<(), std::io::Error> { } } -fn allocate_local_port() -> Result { - let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, 0)) - .map_err(|err| Status::internal(format!("failed to allocate local ssh port: {err}")))?; - listener - .local_addr() - .map(|addr| addr.port()) - .map_err(|err| Status::internal(format!("failed to inspect local ssh port: {err}"))) -} - -async fn port_is_ready(port: u16) -> bool { - TcpStream::connect(SocketAddr::new(Ipv4Addr::LOCALHOST.into(), port)) - .await - .is_ok() -} - async fn guest_ssh_ready(state_dir: &Path) -> bool { let console_log = state_dir.join("rootfs-console.log"); let Ok(contents) = tokio::fs::read_to_string(console_log).await else { @@ -1102,7 +1033,7 @@ mod tests { assert!(env.contains(&"OPENSHELL_ENDPOINT=http://192.168.127.1:8080/".to_string())); assert!(env.contains(&"OPENSHELL_SANDBOX_ID=sandbox-123".to_string())); assert!(env.contains(&format!( - "OPENSHELL_SSH_LISTEN_ADDR=0.0.0.0:{GUEST_SSH_PORT}" + "OPENSHELL_SSH_SOCKET_PATH={GUEST_SSH_SOCKET_PATH}" ))); } @@ -1354,7 +1285,6 @@ mod tests { sandbox_id.to_string(), SandboxRecord { snapshot: sandbox, - ssh_port: 2222, state_dir, process, }, diff --git a/crates/openshell-sandbox/Cargo.toml b/crates/openshell-sandbox/Cargo.toml index 541784ee60..78d8ac741a 100644 --- a/crates/openshell-sandbox/Cargo.toml +++ b/crates/openshell-sandbox/Cargo.toml @@ -53,6 +53,9 @@ webpki-roots = { workspace = true } # HTTP bytes = { workspace = true } +# UUID +uuid = { workspace = true } + # Encoding base64 = { workspace = true } diff --git a/crates/openshell-sandbox/src/grpc_client.rs b/crates/openshell-sandbox/src/grpc_client.rs index c97d1d7925..0af6476c58 100644 --- a/crates/openshell-sandbox/src/grpc_client.rs +++ b/crates/openshell-sandbox/src/grpc_client.rs @@ -33,7 +33,11 @@ async fn connect_channel(endpoint: &str) -> Result { .connect_timeout(Duration::from_secs(10)) .http2_keep_alive_interval(Duration::from_secs(10)) .keep_alive_while_idle(true) - .keep_alive_timeout(Duration::from_secs(20)); + .keep_alive_timeout(Duration::from_secs(20)) + // Match the gateway-side HTTP/2 flow control (see `multiplex.rs`). + // Adaptive sizing lets idle streams stay tiny while bulk + // RelayStream data flows get a BDP-sized window. + .http2_adaptive_window(true); let tls_enabled = endpoint.starts_with("https://"); @@ -74,6 +78,11 @@ async fn connect_channel(endpoint: &str) -> Result { .wrap_err("failed to connect to OpenShell server") } +/// Create a channel to the OpenShell server (public for use by supervisor_session). +pub async fn connect_channel_pub(endpoint: &str) -> Result { + connect_channel(endpoint).await +} + /// Connect to the OpenShell server (mTLS or plaintext based on endpoint scheme). async fn connect(endpoint: &str) -> Result> { let channel = connect_channel(endpoint).await?; diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index cfc6c3051d..49be7dd711 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -21,6 +21,7 @@ pub mod proxy; mod sandbox; mod secrets; mod ssh; +mod supervisor_session; use miette::{IntoDiagnostic, Result}; #[cfg(target_os = "linux")] @@ -209,7 +210,7 @@ pub async fn run_sandbox( openshell_endpoint: Option, policy_rules: Option, policy_data: Option, - ssh_listen_addr: Option, + ssh_socket_path: Option, ssh_handshake_secret: Option, ssh_handshake_skew_secs: u64, _health_check: bool, @@ -609,18 +610,12 @@ pub async fn run_sandbox( } }); - if let Some(listen_addr) = ssh_listen_addr { - let addr: SocketAddr = listen_addr.parse().into_diagnostic()?; + let ssh_socket_path: Option = ssh_socket_path.map(std::path::PathBuf::from); + if let Some(listen_path) = ssh_socket_path.clone() { let policy_clone = policy.clone(); let workdir_clone = workdir.clone(); - let secret = ssh_handshake_secret - .filter(|s| !s.is_empty()) - .ok_or_else(|| { - miette::miette!( - "OPENSHELL_SSH_HANDSHAKE_SECRET is required when SSH is enabled.\n\ - Set --ssh-handshake-secret or the OPENSHELL_SSH_HANDSHAKE_SECRET env var." - ) - })?; + let _ = ssh_handshake_secret; // retained in the signature for compat; unused + let _ = ssh_handshake_skew_secs; let proxy_url = ssh_proxy_url; let netns_fd = ssh_netns_fd; let ca_paths = ca_file_paths.clone(); @@ -630,12 +625,10 @@ pub async fn run_sandbox( tokio::spawn(async move { if let Err(err) = ssh::run_ssh_server( - addr, + listen_path, ssh_ready_tx, policy_clone, workdir_clone, - secret, - ssh_handshake_skew_secs, netns_fd, proxy_url, ca_paths, @@ -684,6 +677,18 @@ pub async fn run_sandbox( } } + // Spawn the persistent supervisor session if we have a gateway endpoint + // and sandbox identity. The session provides relay channels for SSH + // connect and ExecSandbox through the gateway. + if let (Some(endpoint), Some(id), Some(socket)) = ( + openshell_endpoint.as_ref(), + sandbox_id.as_ref(), + ssh_socket_path.as_ref(), + ) { + supervisor_session::spawn(endpoint.clone(), id.clone(), socket.clone()); + info!("supervisor session task spawned"); + } + #[cfg(target_os = "linux")] let mut handle = ProcessHandle::spawn( program, diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index ea70c79cae..b6b8ac24a1 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -69,9 +69,11 @@ struct Args { #[arg(long, default_value = "warn", env = "OPENSHELL_LOG_LEVEL")] log_level: String, - /// SSH listen address for sandbox access. - #[arg(long, env = "OPENSHELL_SSH_LISTEN_ADDR")] - ssh_listen_addr: Option, + /// Filesystem path to the Unix socket the embedded SSH daemon binds. + /// The supervisor bridges `RelayStream` traffic from the gateway onto + /// this socket; nothing else should connect to it. + #[arg(long, env = "OPENSHELL_SSH_SOCKET_PATH")] + ssh_socket_path: Option, /// Shared secret for gateway-to-sandbox SSH handshake. #[arg(long, env = "OPENSHELL_SSH_HANDSHAKE_SECRET")] @@ -226,7 +228,7 @@ fn main() -> Result<()> { args.openshell_endpoint, args.policy_rules, args.policy_data, - args.ssh_listen_addr, + args.ssh_socket_path, args.ssh_handshake_secret, args.ssh_handshake_skew_secs, args.health_check, diff --git a/crates/openshell-sandbox/src/ssh.rs b/crates/openshell-sandbox/src/ssh.rs index 3c26f80611..a8a402b3a6 100644 --- a/crates/openshell-sandbox/src/ssh.rs +++ b/crates/openshell-sandbox/src/ssh.rs @@ -13,8 +13,7 @@ use miette::{IntoDiagnostic, Result}; use nix::pty::{Winsize, openpty}; use nix::unistd::setsid; use openshell_ocsf::{ - ActionId, ActivityId, AuthTypeId, ConfidenceId, DetectionFindingBuilder, DispositionId, - FindingInfo, SeverityId, SshActivityBuilder, StatusId, ocsf_emit, + ActionId, ActivityId, DispositionId, SeverityId, SshActivityBuilder, StatusId, ocsf_emit, }; use rand_core::OsRng; use russh::keys::{Algorithm, PrivateKey}; @@ -22,33 +21,22 @@ use russh::server::{Auth, Handle, Session}; use russh::{ChannelId, CryptoVec}; use std::collections::HashMap; use std::io::{Read, Write}; -use std::net::SocketAddr; use std::os::fd::{AsRawFd, RawFd}; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; -use std::sync::{Arc, Mutex, mpsc}; -use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::net::TcpListener; +use std::sync::{Arc, mpsc}; +use std::time::Duration; +use tokio::net::UnixListener; use tracing::warn; -const PREFACE_MAGIC: &str = "NSSH1"; -#[cfg(test)] -const SSH_HANDSHAKE_SECRET_ENV: &str = "OPENSHELL_SSH_HANDSHAKE_SECRET"; - -/// A time-bounded set of nonces used to detect replayed NSSH1 handshakes. -/// Each entry records the `Instant` it was inserted; a background reaper task -/// periodically evicts entries older than the handshake skew window. -type NonceCache = Arc>>; - /// Perform SSH server initialization: generate a host key, build the config, -/// and bind the TCP listener. Extracted so that startup errors can be forwarded -/// through the readiness channel rather than being silently logged. +/// and bind the Unix socket listener. Extracted so that startup errors can be +/// forwarded through the readiness channel rather than being silently logged. async fn ssh_server_init( - listen_addr: SocketAddr, + listen_path: &Path, ca_file_paths: &Option<(PathBuf, PathBuf)>, ) -> Result<( - TcpListener, + UnixListener, Arc, Option>, )> { @@ -63,14 +51,43 @@ async fn ssh_server_init( let config = Arc::new(config); let ca_paths = ca_file_paths.as_ref().map(|p| Arc::new(p.clone())); - let listener = TcpListener::bind(listen_addr).await.into_diagnostic()?; + + // Ensure the parent directory exists and is root-owned with 0700 + // permissions. The sandbox entrypoint runs as an unprivileged user; it + // must not be able to enter this directory and connect to the socket. + if let Some(parent) = listen_path.parent() { + std::fs::create_dir_all(parent).into_diagnostic()?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let perms = std::fs::Permissions::from_mode(0o700); + std::fs::set_permissions(parent, perms).into_diagnostic()?; + } + } + + // Remove any stale socket from a previous run before binding. + if listen_path.exists() { + std::fs::remove_file(listen_path).into_diagnostic()?; + } + let listener = UnixListener::bind(listen_path).into_diagnostic()?; + + // Tighten permissions so only the supervisor (root) can connect. The + // sandbox entrypoint runs as an unprivileged user and must not be able to + // dial the SSH daemon directly — all access goes through the relay from + // the gateway. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let perms = std::fs::Permissions::from_mode(0o600); + std::fs::set_permissions(listen_path, perms).into_diagnostic()?; + } + ocsf_emit!( SshActivityBuilder::new(crate::ocsf_ctx()) .activity(ActivityId::Listen) .severity(SeverityId::Informational) .status(StatusId::Success) - .src_endpoint_addr(listen_addr.ip(), listen_addr.port()) - .message(format!("SSH server listening on {listen_addr}")) + .message(format!("SSH server listening on {}", listen_path.display())) .build() ); @@ -79,18 +96,16 @@ async fn ssh_server_init( #[allow(clippy::too_many_arguments)] pub async fn run_ssh_server( - listen_addr: SocketAddr, + listen_path: PathBuf, ready_tx: tokio::sync::oneshot::Sender>, policy: SandboxPolicy, workdir: Option, - handshake_secret: String, - handshake_skew_secs: u64, netns_fd: Option, proxy_url: Option, ca_file_paths: Option<(PathBuf, PathBuf)>, provider_env: HashMap, ) -> Result<()> { - let (listener, config, ca_paths) = match ssh_server_init(listen_addr, &ca_file_paths).await { + let (listener, config, ca_paths) = match ssh_server_init(&listen_path, &ca_file_paths).await { Ok(v) => { // Signal that the SSH server has bound the socket and is ready to // accept connections. The parent task awaits this before spawning @@ -105,49 +120,25 @@ pub async fn run_ssh_server( } }; - // Nonce cache for replay detection. Entries are evicted by a background - // reaper once they exceed the handshake skew window. - let nonce_cache: NonceCache = Arc::new(Mutex::new(HashMap::new())); - - // Background task that periodically purges expired nonces. - let reaper_cache = nonce_cache.clone(); - let ttl = Duration::from_secs(handshake_skew_secs); - tokio::spawn(async move { - let mut interval = tokio::time::interval(Duration::from_secs(60)); - loop { - interval.tick().await; - if let Ok(mut cache) = reaper_cache.lock() { - cache.retain(|_, inserted| inserted.elapsed() < ttl); - } - } - }); - loop { - let (stream, peer) = listener.accept().await.into_diagnostic()?; - stream.set_nodelay(true).into_diagnostic()?; + let (stream, _peer) = listener.accept().await.into_diagnostic()?; let config = config.clone(); let policy = policy.clone(); let workdir = workdir.clone(); - let secret = handshake_secret.clone(); let proxy_url = proxy_url.clone(); let ca_paths = ca_paths.clone(); let provider_env = provider_env.clone(); - let nonce_cache = nonce_cache.clone(); tokio::spawn(async move { if let Err(err) = handle_connection( stream, - peer, config, policy, workdir, - &secret, - handshake_skew_secs, netns_fd, proxy_url, ca_paths, provider_env, - &nonce_cache, ) .await { @@ -166,41 +157,18 @@ pub async fn run_ssh_server( #[allow(clippy::too_many_arguments)] async fn handle_connection( - mut stream: tokio::net::TcpStream, - peer: SocketAddr, + stream: tokio::net::UnixStream, config: Arc, policy: SandboxPolicy, workdir: Option, - secret: &str, - handshake_skew_secs: u64, netns_fd: Option, proxy_url: Option, ca_file_paths: Option>, provider_env: HashMap, - nonce_cache: &NonceCache, ) -> Result<()> { - tracing::debug!(peer = %peer, "SSH connection: reading handshake preface"); - let mut line = String::new(); - read_line(&mut stream, &mut line).await?; - tracing::debug!(peer = %peer, preface_len = line.len(), "SSH connection: preface received, verifying"); - if !verify_preface(&line, secret, handshake_skew_secs, nonce_cache)? { - ocsf_emit!( - SshActivityBuilder::new(crate::ocsf_ctx()) - .activity(ActivityId::Open) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .src_endpoint_addr(peer.ip(), peer.port()) - .message(format!( - "SSH connection: handshake verification failed from {peer}" - )) - .build() - ); - let _ = stream.write_all(b"ERR\n").await; - return Ok(()); - } - stream.write_all(b"OK\n").await.into_diagnostic()?; + // Access is gated by the Unix-socket filesystem permissions (root-only), + // not by an application-level preface. The supervisor bridges the + // gateway's RelayStream directly into this socket. ocsf_emit!( SshActivityBuilder::new(crate::ocsf_ctx()) .activity(ActivityId::Open) @@ -208,9 +176,7 @@ async fn handle_connection( .disposition(DispositionId::Allowed) .severity(SeverityId::Informational) .status(StatusId::Success) - .src_endpoint_addr(peer.ip(), peer.port()) - .auth_type(AuthTypeId::Other, "NSSH1") - .message(format!("SSH handshake accepted from {peer}")) + .message("SSH connection accepted on supervisor Unix socket") .build() ); @@ -228,107 +194,6 @@ async fn handle_connection( Ok(()) } -async fn read_line(stream: &mut tokio::net::TcpStream, buf: &mut String) -> Result<()> { - let mut bytes = Vec::new(); - loop { - let mut byte = [0u8; 1]; - let n = stream.read(&mut byte).await.into_diagnostic()?; - if n == 0 { - break; - } - if byte[0] == b'\n' { - break; - } - bytes.push(byte[0]); - if bytes.len() > 1024 { - break; - } - } - *buf = String::from_utf8_lossy(&bytes).to_string(); - Ok(()) -} - -fn verify_preface( - line: &str, - secret: &str, - handshake_skew_secs: u64, - nonce_cache: &NonceCache, -) -> Result { - let parts: Vec<&str> = line.split_whitespace().collect(); - if parts.len() != 5 || parts[0] != PREFACE_MAGIC { - return Ok(false); - } - let token = parts[1]; - let timestamp: i64 = parts[2].parse().unwrap_or(0); - let nonce = parts[3]; - let signature = parts[4]; - - let now = i64::try_from( - SystemTime::now() - .duration_since(UNIX_EPOCH) - .into_diagnostic()? - .as_secs(), - ) - .into_diagnostic()?; - let skew = (now - timestamp).unsigned_abs(); - if skew > handshake_skew_secs { - return Ok(false); - } - - let payload = format!("{token}|{timestamp}|{nonce}"); - let expected = hmac_sha256(secret.as_bytes(), payload.as_bytes()); - if signature != expected { - return Ok(false); - } - - // Reject replayed nonces. The cache is bounded by the reaper task which - // evicts entries older than `handshake_skew_secs`. - let mut cache = nonce_cache - .lock() - .map_err(|_| miette::miette!("nonce cache lock poisoned"))?; - if cache.contains_key(nonce) { - ocsf_emit!( - SshActivityBuilder::new(crate::ocsf_ctx()) - .activity(ActivityId::Other) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::High) - .auth_type(AuthTypeId::Other, "NSSH1") - .message(format!("NSSH1 nonce replay detected: {nonce}")) - .build() - ); - ocsf_emit!( - DetectionFindingBuilder::new(crate::ocsf_ctx()) - .activity(ActivityId::Open) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::High) - .is_alert(true) - .confidence(ConfidenceId::High) - .finding_info(FindingInfo::new( - "nssh1-nonce-replay", - "NSSH1 Nonce Replay Attack" - )) - .evidence("nonce", nonce) - .build() - ); - return Ok(false); - } - cache.insert(nonce.to_string(), Instant::now()); - - Ok(true) -} - -fn hmac_sha256(key: &[u8], data: &[u8]) -> String { - use hmac::{Hmac, Mac}; - use sha2::Sha256; - - let mut mac = Hmac::::new_from_slice(key).expect("hmac key"); - mac.update(data); - let result = mac.finalize().into_bytes(); - hex::encode(result) -} - /// Per-channel state for tracking PTY resources and I/O senders. /// /// Each SSH channel gets its own PTY master (if a PTY was requested) and input @@ -1424,136 +1289,6 @@ mod tests { ); } - // ----------------------------------------------------------------------- - // verify_preface tests - // ----------------------------------------------------------------------- - - /// Build a valid NSSH1 preface line with the given parameters. - fn build_preface(token: &str, secret: &str, nonce: &str, timestamp: i64) -> String { - let payload = format!("{token}|{timestamp}|{nonce}"); - let signature = hmac_sha256(secret.as_bytes(), payload.as_bytes()); - format!("{PREFACE_MAGIC} {token} {timestamp} {nonce} {signature}") - } - - fn fresh_nonce_cache() -> NonceCache { - Arc::new(Mutex::new(HashMap::new())) - } - - fn current_timestamp() -> i64 { - i64::try_from( - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_secs(), - ) - .unwrap() - } - - #[test] - fn verify_preface_accepts_valid_preface() { - let secret = "test-secret-key"; - let nonce = "unique-nonce-1"; - let ts = current_timestamp(); - let line = build_preface("tok1", secret, nonce, ts); - let cache = fresh_nonce_cache(); - - assert!(verify_preface(&line, secret, 300, &cache).unwrap()); - } - - #[test] - fn verify_preface_rejects_replayed_nonce() { - let secret = "test-secret-key"; - let nonce = "replay-nonce"; - let ts = current_timestamp(); - let line = build_preface("tok1", secret, nonce, ts); - let cache = fresh_nonce_cache(); - - // First attempt should succeed. - assert!(verify_preface(&line, secret, 300, &cache).unwrap()); - // Second attempt with the same nonce should be rejected. - assert!(!verify_preface(&line, secret, 300, &cache).unwrap()); - } - - #[test] - fn verify_preface_rejects_expired_timestamp() { - let secret = "test-secret-key"; - let nonce = "expired-nonce"; - // Timestamp 600 seconds in the past, with a 300-second skew window. - let ts = current_timestamp() - 600; - let line = build_preface("tok1", secret, nonce, ts); - let cache = fresh_nonce_cache(); - - assert!(!verify_preface(&line, secret, 300, &cache).unwrap()); - } - - #[test] - fn verify_preface_rejects_invalid_hmac() { - let secret = "test-secret-key"; - let nonce = "hmac-nonce"; - let ts = current_timestamp(); - // Build with the correct secret, then verify with the wrong one. - let line = build_preface("tok1", secret, nonce, ts); - let cache = fresh_nonce_cache(); - - assert!(!verify_preface(&line, "wrong-secret", 300, &cache).unwrap()); - } - - #[test] - fn verify_preface_rejects_malformed_input() { - let cache = fresh_nonce_cache(); - - // Too few parts. - assert!(!verify_preface("NSSH1 tok1 123", "s", 300, &cache).unwrap()); - // Wrong magic. - assert!(!verify_preface("NSSH2 tok1 123 nonce sig", "s", 300, &cache).unwrap()); - // Empty string. - assert!(!verify_preface("", "s", 300, &cache).unwrap()); - } - - #[test] - fn verify_preface_distinct_nonces_both_accepted() { - let secret = "test-secret-key"; - let ts = current_timestamp(); - let cache = fresh_nonce_cache(); - - let line1 = build_preface("tok1", secret, "nonce-a", ts); - let line2 = build_preface("tok1", secret, "nonce-b", ts); - - assert!(verify_preface(&line1, secret, 300, &cache).unwrap()); - assert!(verify_preface(&line2, secret, 300, &cache).unwrap()); - } - - #[test] - fn apply_child_env_keeps_handshake_secret_out_of_ssh_children() { - let mut cmd = Command::new("/usr/bin/env"); - cmd.env(SSH_HANDSHAKE_SECRET_ENV, "should-not-leak") - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::null()); - - let provider_env = std::iter::once(( - "ANTHROPIC_API_KEY".to_string(), - "openshell:resolve:env:ANTHROPIC_API_KEY".to_string(), - )) - .collect(); - - apply_child_env( - &mut cmd, - "/sandbox", - "sandbox", - "dumb", - None, - None, - &provider_env, - ); - - let output = cmd.output().expect("spawn env"); - let stdout = String::from_utf8(output.stdout).expect("utf8"); - - assert!(!stdout.contains(SSH_HANDSHAKE_SECRET_ENV)); - assert!(stdout.contains("ANTHROPIC_API_KEY=openshell:resolve:env:ANTHROPIC_API_KEY")); - } - // ----------------------------------------------------------------------- // SEC-007: is_loopback_host tests // ----------------------------------------------------------------------- diff --git a/crates/openshell-sandbox/src/supervisor_session.rs b/crates/openshell-sandbox/src/supervisor_session.rs new file mode 100644 index 0000000000..490a0cba77 --- /dev/null +++ b/crates/openshell-sandbox/src/supervisor_session.rs @@ -0,0 +1,570 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Persistent supervisor-to-gateway session. +//! +//! Maintains a long-lived `ConnectSupervisor` bidirectional gRPC stream to the +//! gateway. When the gateway sends `RelayOpen`, the supervisor initiates a +//! `RelayStream` gRPC call (a new HTTP/2 stream multiplexed over the same +//! TCP+TLS connection as the control stream) and bridges it to the local SSH +//! daemon. The supervisor is a dumb byte bridge — it has no protocol awareness +//! of the SSH or NSSH1 bytes flowing through. + +use std::time::Duration; + +use openshell_core::proto::open_shell_client::OpenShellClient; +use openshell_core::proto::{ + GatewayMessage, RelayFrame, RelayInit, SupervisorHeartbeat, SupervisorHello, SupervisorMessage, + gateway_message, supervisor_message, +}; +use openshell_ocsf::{ + ActivityId, Endpoint, NetworkActivityBuilder, OcsfEvent, SandboxContext, SeverityId, StatusId, + ocsf_emit, +}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::sync::mpsc; +use tokio_stream::StreamExt; +use tonic::transport::Channel; +use tracing::{debug, warn}; + +use crate::grpc_client; + +const INITIAL_BACKOFF: Duration = Duration::from_secs(1); +const MAX_BACKOFF: Duration = Duration::from_secs(30); + +/// Parse a gRPC endpoint URI into an OCSF `Endpoint` (host + port). Falls back +/// to treating the whole string as a domain if parsing fails. +fn ocsf_gateway_endpoint(endpoint: &str) -> Endpoint { + let without_scheme = endpoint + .split_once("://") + .map_or(endpoint, |(_, rest)| rest); + let host_and_port = without_scheme.split('/').next().unwrap_or(without_scheme); + if let Some((host, port)) = host_and_port.rsplit_once(':') + && let Ok(port) = port.parse::() + { + return Endpoint::from_domain(host, port); + } + Endpoint::from_domain(host_and_port, 0) +} + +fn session_established_event( + ctx: &SandboxContext, + endpoint: &str, + session_id: &str, + heartbeat_secs: u32, +) -> OcsfEvent { + NetworkActivityBuilder::new(ctx) + .activity(ActivityId::Open) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .dst_endpoint(ocsf_gateway_endpoint(endpoint)) + .message(format!( + "supervisor session established (session_id={session_id}, heartbeat_secs={heartbeat_secs})" + )) + .build() +} + +fn session_closed_event(ctx: &SandboxContext, endpoint: &str, sandbox_id: &str) -> OcsfEvent { + NetworkActivityBuilder::new(ctx) + .activity(ActivityId::Close) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .dst_endpoint(ocsf_gateway_endpoint(endpoint)) + .message(format!("supervisor session ended cleanly ({sandbox_id})")) + .build() +} + +fn session_failed_event( + ctx: &SandboxContext, + endpoint: &str, + attempt: u64, + error: &str, +) -> OcsfEvent { + NetworkActivityBuilder::new(ctx) + .activity(ActivityId::Fail) + .severity(SeverityId::Low) + .status(StatusId::Failure) + .dst_endpoint(ocsf_gateway_endpoint(endpoint)) + .message(format!( + "supervisor session failed, reconnecting (attempt {attempt}): {error}" + )) + .build() +} + +fn relay_open_event(ctx: &SandboxContext, channel_id: &str) -> OcsfEvent { + NetworkActivityBuilder::new(ctx) + .activity(ActivityId::Open) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .message(format!("relay open (channel_id={channel_id})")) + .build() +} + +fn relay_closed_event(ctx: &SandboxContext, channel_id: &str) -> OcsfEvent { + NetworkActivityBuilder::new(ctx) + .activity(ActivityId::Close) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .message(format!("relay closed (channel_id={channel_id})")) + .build() +} + +fn relay_failed_event(ctx: &SandboxContext, channel_id: &str, error: &str) -> OcsfEvent { + NetworkActivityBuilder::new(ctx) + .activity(ActivityId::Fail) + .severity(SeverityId::Low) + .status(StatusId::Failure) + .message(format!( + "relay bridge failed (channel_id={channel_id}): {error}" + )) + .build() +} + +fn relay_close_from_gateway_event( + ctx: &SandboxContext, + channel_id: &str, + reason: &str, +) -> OcsfEvent { + NetworkActivityBuilder::new(ctx) + .activity(ActivityId::Close) + .severity(SeverityId::Informational) + .message(format!( + "relay close from gateway (channel_id={channel_id}, reason={reason})" + )) + .build() +} + +/// Size of chunks read from the local SSH socket when forwarding bytes back +/// to the gateway over the gRPC response stream. 16 KiB matches the default +/// HTTP/2 frame size so each `RelayFrame::data` fits in one frame. +const RELAY_CHUNK_SIZE: usize = 16 * 1024; + +fn map_stream_message( + message: Result, tonic::Status>, + eof_error: &'static str, +) -> Result> { + match message { + Ok(Some(msg)) => Ok(msg), + Ok(None) => Err(eof_error.into()), + Err(e) => Err(format!("stream error: {e}").into()), + } +} + +/// Spawn the supervisor session task. +/// +/// The task runs for the lifetime of the sandbox process, reconnecting with +/// exponential backoff on failures. +pub fn spawn( + endpoint: String, + sandbox_id: String, + ssh_socket_path: std::path::PathBuf, +) -> tokio::task::JoinHandle<()> { + tokio::spawn(run_session_loop(endpoint, sandbox_id, ssh_socket_path)) +} + +async fn run_session_loop( + endpoint: String, + sandbox_id: String, + ssh_socket_path: std::path::PathBuf, +) { + let mut backoff = INITIAL_BACKOFF; + let mut attempt: u64 = 0; + + loop { + attempt += 1; + + match run_single_session(&endpoint, &sandbox_id, &ssh_socket_path).await { + Ok(()) => { + let event = session_closed_event(crate::ocsf_ctx(), &endpoint, &sandbox_id); + ocsf_emit!(event); + break; + } + Err(e) => { + let event = + session_failed_event(crate::ocsf_ctx(), &endpoint, attempt, &e.to_string()); + ocsf_emit!(event); + tokio::time::sleep(backoff).await; + backoff = (backoff * 2).min(MAX_BACKOFF); + } + } + } +} + +async fn run_single_session( + endpoint: &str, + sandbox_id: &str, + ssh_socket_path: &std::path::Path, +) -> Result<(), Box> { + // Connect to the gateway. The same `Channel` is used for both the + // long-lived control stream and all data-plane `RelayStream` calls, so + // every relay rides the same TCP+TLS+HTTP/2 connection — no new TLS + // handshake per relay. + let channel = grpc_client::connect_channel_pub(endpoint) + .await + .map_err(|e| format!("connect failed: {e}"))?; + let mut client = OpenShellClient::new(channel.clone()); + + // Create the outbound message stream. + let (tx, rx) = mpsc::channel::(64); + let outbound = tokio_stream::wrappers::ReceiverStream::new(rx); + + // Send hello as the first message. + let instance_id = uuid::Uuid::new_v4().to_string(); + tx.send(SupervisorMessage { + payload: Some(supervisor_message::Payload::Hello(SupervisorHello { + sandbox_id: sandbox_id.to_string(), + instance_id: instance_id.clone(), + })), + }) + .await + .map_err(|_| "failed to queue hello")?; + + // Open the bidirectional stream. + let response = client + .connect_supervisor(outbound) + .await + .map_err(|e| format!("connect_supervisor RPC failed: {e}"))?; + let mut inbound = response.into_inner(); + + // Wait for SessionAccepted. + let accepted = match map_stream_message( + inbound.message().await, + "stream closed before session accepted", + )? + .payload + { + Some(gateway_message::Payload::SessionAccepted(a)) => a, + Some(gateway_message::Payload::SessionRejected(r)) => { + return Err(format!("session rejected: {}", r.reason).into()); + } + _ => return Err("expected SessionAccepted or SessionRejected".into()), + }; + + let heartbeat_secs = accepted.heartbeat_interval_secs.max(5); + let event = session_established_event( + crate::ocsf_ctx(), + endpoint, + &accepted.session_id, + heartbeat_secs, + ); + ocsf_emit!(event); + + // Main loop: receive gateway messages + send heartbeats. + let mut heartbeat_interval = + tokio::time::interval(Duration::from_secs(u64::from(heartbeat_secs))); + heartbeat_interval.tick().await; // skip immediate tick + + loop { + tokio::select! { + msg = inbound.message() => { + let msg = map_stream_message(msg, "gateway closed stream")?; + handle_gateway_message( + &msg, + sandbox_id, + ssh_socket_path, + &channel, + ); + } + _ = heartbeat_interval.tick() => { + let hb = SupervisorMessage { + payload: Some(supervisor_message::Payload::Heartbeat( + SupervisorHeartbeat {}, + )), + }; + if tx.send(hb).await.is_err() { + return Err("outbound channel closed".into()); + } + } + } + } +} + +fn handle_gateway_message( + msg: &GatewayMessage, + sandbox_id: &str, + ssh_socket_path: &std::path::Path, + channel: &Channel, +) { + match &msg.payload { + Some(gateway_message::Payload::Heartbeat(_)) => { + // Gateway heartbeat — nothing to do. + } + Some(gateway_message::Payload::RelayOpen(open)) => { + let channel_id = open.channel_id.clone(); + let sandbox_id = sandbox_id.to_string(); + let channel = channel.clone(); + let ssh_socket_path = ssh_socket_path.to_path_buf(); + + let event = relay_open_event(crate::ocsf_ctx(), &channel_id); + ocsf_emit!(event); + + tokio::spawn(async move { + match handle_relay_open(&channel_id, &ssh_socket_path, channel).await { + Ok(()) => { + let event = relay_closed_event(crate::ocsf_ctx(), &channel_id); + ocsf_emit!(event); + } + Err(e) => { + let event = + relay_failed_event(crate::ocsf_ctx(), &channel_id, &e.to_string()); + ocsf_emit!(event); + warn!( + sandbox_id = %sandbox_id, + channel_id = %channel_id, + error = %e, + "supervisor session: relay bridge failed" + ); + } + } + }); + } + Some(gateway_message::Payload::RelayClose(close)) => { + let event = + relay_close_from_gateway_event(crate::ocsf_ctx(), &close.channel_id, &close.reason); + ocsf_emit!(event); + } + _ => { + warn!(sandbox_id = %sandbox_id, "supervisor session: unexpected gateway message"); + } + } +} + +/// Handle a `RelayOpen` by initiating a `RelayStream` RPC on the gateway and +/// bridging that stream to the local SSH daemon. +/// +/// This opens a new HTTP/2 stream on the existing `Channel` — no new TCP or +/// TLS handshake. The first `RelayFrame` we send is a `RelayInit`; subsequent +/// frames carry raw SSH bytes in `data`. +async fn handle_relay_open( + channel_id: &str, + ssh_socket_path: &std::path::Path, + channel: Channel, +) -> Result<(), Box> { + let mut client = OpenShellClient::new(channel); + + // Outbound chunks to the gateway. + let (out_tx, out_rx) = mpsc::channel::(16); + let outbound = tokio_stream::wrappers::ReceiverStream::new(out_rx); + + // First frame: identify the channel. + out_tx + .send(RelayFrame { + payload: Some(openshell_core::proto::relay_frame::Payload::Init( + RelayInit { + channel_id: channel_id.to_string(), + }, + )), + }) + .await + .map_err(|_| "outbound channel closed before init")?; + + // Initiate the RPC. This rides the existing HTTP/2 connection. + let response = client + .relay_stream(outbound) + .await + .map_err(|e| format!("relay_stream RPC failed: {e}"))?; + let mut inbound = response.into_inner(); + + // Connect to the local SSH daemon on its Unix socket. + let ssh = tokio::net::UnixStream::connect(ssh_socket_path).await?; + let (mut ssh_r, mut ssh_w) = ssh.into_split(); + + debug!( + channel_id = %channel_id, + socket = %ssh_socket_path.display(), + "relay bridge: connected to local SSH daemon" + ); + + // SSH → gRPC (out_tx): read local SSH, forward as `RelayFrame::data`. + let out_tx_writer = out_tx.clone(); + let ssh_to_grpc = tokio::spawn(async move { + let mut buf = vec![0u8; RELAY_CHUNK_SIZE]; + loop { + match ssh_r.read(&mut buf).await { + Ok(0) | Err(_) => break, + Ok(n) => { + let chunk = RelayFrame { + payload: Some(openshell_core::proto::relay_frame::Payload::Data( + buf[..n].to_vec(), + )), + }; + if out_tx_writer.send(chunk).await.is_err() { + break; + } + } + } + } + }); + + // gRPC (inbound) → SSH: drain inbound chunks into the local SSH socket. + let mut inbound_err: Option = None; + while let Some(next) = inbound.next().await { + match next { + Ok(frame) => { + let Some(openshell_core::proto::relay_frame::Payload::Data(data)) = frame.payload + else { + inbound_err = Some("relay inbound received non-data frame".to_string()); + break; + }; + if data.is_empty() { + continue; + } + if let Err(e) = ssh_w.write_all(&data).await { + inbound_err = Some(format!("write to ssh failed: {e}")); + break; + } + } + Err(e) => { + inbound_err = Some(format!("relay inbound errored: {e}")); + break; + } + } + } + + // Half-close the SSH socket's write side so the daemon sees EOF. + let _ = ssh_w.shutdown().await; + + // Dropping out_tx closes the outbound gRPC stream, letting the gateway + // observe EOF on its side too. + drop(out_tx); + let _ = ssh_to_grpc.await; + + if let Some(e) = inbound_err { + return Err(e.into()); + } + Ok(()) +} + +#[cfg(test)] +mod ocsf_event_tests { + use super::*; + + fn ctx() -> SandboxContext { + SandboxContext { + sandbox_id: "sbx-1".into(), + sandbox_name: "sandbox".into(), + container_image: "img".into(), + hostname: "host".into(), + product_version: "0.0.1".into(), + proxy_ip: "127.0.0.1".parse().unwrap(), + proxy_port: 3128, + } + } + + #[test] + fn gateway_endpoint_parses_https_with_port() { + let e = ocsf_gateway_endpoint("https://gateway.openshell:8443"); + assert_eq!(e.domain.as_deref(), Some("gateway.openshell")); + assert_eq!(e.port, Some(8443)); + } + + #[test] + fn gateway_endpoint_parses_http_with_port_and_path() { + let e = ocsf_gateway_endpoint("http://gw:7000/grpc"); + assert_eq!(e.domain.as_deref(), Some("gw")); + assert_eq!(e.port, Some(7000)); + } + + #[test] + fn gateway_endpoint_falls_back_without_port() { + let e = ocsf_gateway_endpoint("gateway.openshell"); + assert_eq!(e.domain.as_deref(), Some("gateway.openshell")); + assert_eq!(e.port, Some(0)); + } + + fn network_activity(event: &OcsfEvent) -> &openshell_ocsf::NetworkActivityEvent { + match event { + OcsfEvent::NetworkActivity(n) => n, + other => panic!("expected NetworkActivity, got {other:?}"), + } + } + + #[test] + fn session_established_emits_network_open_success() { + let event = session_established_event(&ctx(), "https://gw:443", "sess-1", 30); + let na = network_activity(&event); + assert_eq!(na.base.activity_id, ActivityId::Open.as_u8()); + assert_eq!(na.base.severity, SeverityId::Informational); + assert_eq!(na.base.status, Some(StatusId::Success)); + assert_eq!( + na.dst_endpoint.as_ref().and_then(|e| e.domain.as_deref()), + Some("gw") + ); + let msg = na.base.message.as_deref().unwrap_or_default(); + assert!(msg.contains("sess-1"), "message missing session_id: {msg}"); + assert!(msg.contains("heartbeat_secs=30"), "message: {msg}"); + } + + #[test] + fn session_closed_emits_network_close_success() { + let event = session_closed_event(&ctx(), "https://gw:443", "sbx-1"); + let na = network_activity(&event); + assert_eq!(na.base.activity_id, ActivityId::Close.as_u8()); + assert_eq!(na.base.severity, SeverityId::Informational); + assert_eq!(na.base.status, Some(StatusId::Success)); + } + + #[test] + fn session_failed_emits_network_fail_low() { + let event = session_failed_event(&ctx(), "https://gw:443", 3, "connect refused"); + let na = network_activity(&event); + assert_eq!(na.base.activity_id, ActivityId::Fail.as_u8()); + assert_eq!(na.base.severity, SeverityId::Low); + assert_eq!(na.base.status, Some(StatusId::Failure)); + let msg = na.base.message.as_deref().unwrap_or_default(); + assert!(msg.contains("attempt 3"), "message: {msg}"); + assert!(msg.contains("connect refused"), "message: {msg}"); + } + + #[test] + fn relay_open_emits_network_open_success() { + let event = relay_open_event(&ctx(), "ch-42"); + let na = network_activity(&event); + assert_eq!(na.base.activity_id, ActivityId::Open.as_u8()); + assert_eq!(na.base.severity, SeverityId::Informational); + assert!( + na.base + .message + .as_deref() + .unwrap_or_default() + .contains("ch-42") + ); + } + + #[test] + fn relay_closed_emits_network_close_success() { + let event = relay_closed_event(&ctx(), "ch-42"); + let na = network_activity(&event); + assert_eq!(na.base.activity_id, ActivityId::Close.as_u8()); + assert_eq!(na.base.status, Some(StatusId::Success)); + } + + #[test] + fn relay_failed_emits_network_fail_low() { + let event = relay_failed_event(&ctx(), "ch-42", "write to ssh failed"); + let na = network_activity(&event); + assert_eq!(na.base.activity_id, ActivityId::Fail.as_u8()); + assert_eq!(na.base.severity, SeverityId::Low); + assert_eq!(na.base.status, Some(StatusId::Failure)); + let msg = na.base.message.as_deref().unwrap_or_default(); + assert!(msg.contains("ch-42"), "message: {msg}"); + assert!(msg.contains("write to ssh failed"), "message: {msg}"); + } + + #[test] + fn relay_close_from_gateway_is_network_close_informational() { + let event = relay_close_from_gateway_event(&ctx(), "ch-42", "sandbox deleted"); + let na = network_activity(&event); + assert_eq!(na.base.activity_id, ActivityId::Close.as_u8()); + assert_eq!(na.base.severity, SeverityId::Informational); + let msg = na.base.message.as_deref().unwrap_or_default(); + assert!(msg.contains("sandbox deleted"), "message: {msg}"); + } + + #[test] + fn map_stream_message_treats_eof_as_reconnectable_error() { + let err = map_stream_message::(Ok(None), "gateway closed stream") + .expect_err("eof should force reconnect"); + assert_eq!(err.to_string(), "gateway closed stream"); + } +} diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index a36ad05e7e..8eff8c8578 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -91,10 +91,6 @@ struct Args { )] ssh_connect_path: String, - /// SSH port inside sandbox pods. - #[arg(long, env = "OPENSHELL_SANDBOX_SSH_PORT", default_value_t = 2222)] - sandbox_ssh_port: u16, - /// Shared secret for gateway-to-sandbox SSH handshake. #[arg(long, env = "OPENSHELL_SSH_HANDSHAKE_SECRET")] ssh_handshake_secret: Option, @@ -237,7 +233,6 @@ async fn run_from_args(args: Args) -> Result<()> { .with_ssh_gateway_host(args.ssh_gateway_host) .with_ssh_gateway_port(args.ssh_gateway_port) .with_ssh_connect_path(args.ssh_connect_path) - .with_sandbox_ssh_port(args.sandbox_ssh_port) .with_ssh_handshake_skew_secs(args.ssh_handshake_skew_secs); if let Some(image) = args.sandbox_image { diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 6b55740160..95ffbfaa49 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -17,9 +17,9 @@ use openshell_core::proto::compute::v1::{ CreateSandboxRequest, DeleteSandboxRequest, DriverCondition, DriverPlatformEvent, DriverResourceRequirements, DriverSandbox, DriverSandboxSpec, DriverSandboxStatus, DriverSandboxTemplate, GetCapabilitiesRequest, GetSandboxRequest, ListSandboxesRequest, - ResolveSandboxEndpointRequest, ResolveSandboxEndpointResponse, ValidateSandboxCreateRequest, - WatchSandboxesEvent, WatchSandboxesRequest, compute_driver_client::ComputeDriverClient, - compute_driver_server::ComputeDriver, sandbox_endpoint, watch_sandboxes_event, + ValidateSandboxCreateRequest, WatchSandboxesEvent, WatchSandboxesRequest, + compute_driver_client::ComputeDriverClient, compute_driver_server::ComputeDriver, + watch_sandboxes_event, }; use openshell_core::proto::{ PlatformEvent, Sandbox, SandboxCondition, SandboxPhase, SandboxSpec, SandboxStatus, @@ -30,7 +30,6 @@ use openshell_driver_kubernetes::{ }; use prost::Message; use std::fmt; -use std::net::IpAddr; use std::pin::Pin; use std::sync::Arc; use std::time::Duration; @@ -59,12 +58,6 @@ pub enum ComputeError { #[error("{0}")] Message(String), } -#[derive(Debug)] -pub enum ResolvedEndpoint { - Ip(IpAddr, u16), - Host(String, u16), -} - #[derive(Debug)] pub(crate) struct ManagedDriverProcess { child: std::sync::Mutex>, @@ -173,14 +166,6 @@ impl ComputeDriver for RemoteComputeDriver { client.delete_sandbox(request).await } - async fn resolve_sandbox_endpoint( - &self, - request: Request, - ) -> Result, Status> { - let mut client = self.client(); - client.resolve_sandbox_endpoint(request).await - } - async fn watch_sandboxes( &self, request: Request, @@ -417,29 +402,6 @@ impl ComputeRuntime { Ok(deleted) } - pub async fn resolve_sandbox_endpoint( - &self, - sandbox: &Sandbox, - ) -> Result { - let driver_sandbox = driver_sandbox_from_public(sandbox); - self.driver - .resolve_sandbox_endpoint(Request::new(ResolveSandboxEndpointRequest { - sandbox: Some(driver_sandbox), - })) - .await - .map(|response| response.into_inner()) - .map_err(|status| match status.code() { - Code::FailedPrecondition => { - Status::failed_precondition(status.message().to_string()) - } - _ => Status::internal(status.message().to_string()), - }) - .and_then(|response| { - resolved_endpoint_from_response(&response) - .map_err(|err| Status::internal(err.to_string())) - }) - } - pub fn spawn_watchers(&self) { let runtime = Arc::new(self.clone()); let watch_runtime = runtime.clone(); @@ -987,30 +949,6 @@ fn decode_sandbox_record(record: &ObjectRecord) -> Result { Sandbox::decode(record.payload.as_slice()).map_err(|e| e.to_string()) } -fn resolved_endpoint_from_response( - response: &ResolveSandboxEndpointResponse, -) -> Result { - let endpoint = response - .endpoint - .as_ref() - .ok_or_else(|| ComputeError::Message("compute driver returned no endpoint".to_string()))?; - let port = u16::try_from(endpoint.port) - .map_err(|_| ComputeError::Message("compute driver returned invalid port".to_string()))?; - - match endpoint.target.as_ref() { - Some(sandbox_endpoint::Target::Ip(ip)) => ip - .parse() - .map(|ip| ResolvedEndpoint::Ip(ip, port)) - .map_err(|e| ComputeError::Message(format!("invalid endpoint IP: {e}"))), - Some(sandbox_endpoint::Target::Host(host)) => { - Ok(ResolvedEndpoint::Host(host.clone(), port)) - } - None => Err(ComputeError::Message( - "compute driver returned endpoint without target".to_string(), - )), - } -} - fn public_status_from_driver(status: &DriverSandboxStatus) -> SandboxStatus { SandboxStatus { sandbox_name: status.sandbox_name.clone(), @@ -1103,8 +1041,7 @@ mod tests { use futures::stream; use openshell_core::proto::compute::v1::{ CreateSandboxResponse, DeleteSandboxResponse, GetCapabilitiesResponse, GetSandboxRequest, - GetSandboxResponse, ResolveSandboxEndpointResponse, SandboxEndpoint, StopSandboxRequest, - StopSandboxResponse, ValidateSandboxCreateResponse, sandbox_endpoint, + GetSandboxResponse, StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateResponse, }; use std::sync::Arc; @@ -1112,7 +1049,6 @@ mod tests { struct TestDriver { listed_sandboxes: Vec, current_sandboxes: Vec, - resolve_precondition: Option, } #[tonic::async_trait] @@ -1205,24 +1141,6 @@ mod tests { })) } - async fn resolve_sandbox_endpoint( - &self, - _request: Request, - ) -> Result, Status> { - if let Some(message) = &self.resolve_precondition { - return Err(Status::failed_precondition(message.clone())); - } - - Ok(tonic::Response::new(ResolveSandboxEndpointResponse { - endpoint: Some(SandboxEndpoint { - target: Some(sandbox_endpoint::Target::Host( - "sandbox.default.svc.cluster.local".to_string(), - )), - port: 2222, - }), - })) - } - async fn watch_sandboxes( &self, _request: Request, @@ -1499,23 +1417,6 @@ mod tests { ); } - #[tokio::test] - async fn resolve_sandbox_endpoint_preserves_precondition_errors() { - let runtime = test_runtime(Arc::new(TestDriver { - resolve_precondition: Some("sandbox agent pod IP is not available".to_string()), - ..Default::default() - })) - .await; - - let err = runtime - .resolve_sandbox_endpoint(&sandbox_record("sb-1", "sandbox-a", SandboxPhase::Ready)) - .await - .expect_err("endpoint resolution should preserve failed-precondition errors"); - - assert_eq!(err.code(), Code::FailedPrecondition); - assert_eq!(err.message(), "sandbox agent pod IP is not available"); - } - #[tokio::test] async fn reconcile_store_with_backend_applies_driver_snapshot() { let runtime = test_runtime(Arc::new(TestDriver { diff --git a/crates/openshell-server/src/grpc/mod.rs b/crates/openshell-server/src/grpc/mod.rs index af60897d1f..9eab56d473 100644 --- a/crates/openshell-server/src/grpc/mod.rs +++ b/crates/openshell-server/src/grpc/mod.rs @@ -14,22 +14,23 @@ use openshell_core::proto::{ CreateProviderRequest, CreateSandboxRequest, CreateSshSessionRequest, CreateSshSessionResponse, DeleteProviderRequest, DeleteProviderResponse, DeleteSandboxRequest, DeleteSandboxResponse, EditDraftChunkRequest, EditDraftChunkResponse, ExecSandboxEvent, ExecSandboxRequest, - GetDraftHistoryRequest, GetDraftHistoryResponse, GetDraftPolicyRequest, GetDraftPolicyResponse, - GetGatewayConfigRequest, GetGatewayConfigResponse, GetProviderRequest, GetSandboxConfigRequest, - GetSandboxConfigResponse, GetSandboxLogsRequest, GetSandboxLogsResponse, - GetSandboxPolicyStatusRequest, GetSandboxPolicyStatusResponse, + GatewayMessage, GetDraftHistoryRequest, GetDraftHistoryResponse, GetDraftPolicyRequest, + GetDraftPolicyResponse, GetGatewayConfigRequest, GetGatewayConfigResponse, GetProviderRequest, + GetSandboxConfigRequest, GetSandboxConfigResponse, GetSandboxLogsRequest, + GetSandboxLogsResponse, GetSandboxPolicyStatusRequest, GetSandboxPolicyStatusResponse, GetSandboxProviderEnvironmentRequest, GetSandboxProviderEnvironmentResponse, GetSandboxRequest, HealthRequest, HealthResponse, ListProvidersRequest, ListProvidersResponse, ListSandboxPoliciesRequest, ListSandboxPoliciesResponse, ListSandboxesRequest, ListSandboxesResponse, ProviderResponse, PushSandboxLogsRequest, PushSandboxLogsResponse, - RejectDraftChunkRequest, RejectDraftChunkResponse, ReportPolicyStatusRequest, + RejectDraftChunkRequest, RejectDraftChunkResponse, RelayFrame, ReportPolicyStatusRequest, ReportPolicyStatusResponse, RevokeSshSessionRequest, RevokeSshSessionResponse, SandboxResponse, SandboxStreamEvent, ServiceStatus, SubmitPolicyAnalysisRequest, SubmitPolicyAnalysisResponse, - UndoDraftChunkRequest, UndoDraftChunkResponse, UpdateConfigRequest, UpdateConfigResponse, - UpdateProviderRequest, WatchSandboxRequest, open_shell_server::OpenShell, + SupervisorMessage, UndoDraftChunkRequest, UndoDraftChunkResponse, UpdateConfigRequest, + UpdateConfigResponse, UpdateProviderRequest, WatchSandboxRequest, open_shell_server::OpenShell, }; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; +use std::pin::Pin; use std::sync::Arc; use tokio_stream::wrappers::ReceiverStream; use tonic::{Request, Response, Status}; @@ -383,6 +384,29 @@ impl OpenShell for OpenShellService { ) -> Result, Status> { policy::handle_get_draft_history(&self.state, request).await } + + // --- Supervisor session --- + + type ConnectSupervisorStream = + Pin> + Send + 'static>>; + + async fn connect_supervisor( + &self, + request: Request>, + ) -> Result, Status> { + crate::supervisor_session::handle_connect_supervisor(&self.state, request).await + } + + type RelayStreamStream = + Pin> + Send + 'static>>; + + async fn relay_stream( + &self, + request: Request>, + ) -> Result, Status> { + crate::supervisor_session::handle_relay_stream(&self.state.supervisor_sessions, request) + .await + } } // --------------------------------------------------------------------------- diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index 0a729c099f..6b01b28f41 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -22,13 +22,11 @@ use openshell_core::proto::{ use openshell_core::proto::{Sandbox, SandboxPhase, SandboxTemplate, SshSession}; use prost::Message; use std::sync::Arc; -use tokio::io::AsyncReadExt; -use tokio::io::AsyncWriteExt; use tokio::net::{TcpListener, TcpStream}; use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; use tonic::{Request, Response, Status}; -use tracing::{debug, info, warn}; +use tracing::{info, warn}; use russh::ChannelMsg; use russh::client::AuthResult; @@ -438,27 +436,55 @@ pub(super) async fn handle_exec_sandbox( return Err(Status::failed_precondition("sandbox is not ready")); } - let (target_host, target_port) = resolve_sandbox_exec_target(state, &sandbox).await?; + // Open a relay channel through the supervisor session. Use a 15s + // session-wait timeout — enough to cover a transient supervisor + // reconnect, but shorter than `/connect/ssh` since `ExecSandbox` is + // typically called during normal operation (not right after create). + let (channel_id, relay_rx) = state + .supervisor_sessions + .open_relay(&sandbox.id, std::time::Duration::from_secs(15)) + .await + .map_err(|e| Status::unavailable(format!("supervisor relay failed: {e}")))?; + let command_str = build_remote_exec_command(&req) .map_err(|e| Status::invalid_argument(format!("command construction failed: {e}")))?; let stdin_payload = req.stdin; let timeout_seconds = req.timeout_seconds; let request_tty = req.tty; let sandbox_id = sandbox.id; - let handshake_secret = state.config.ssh_handshake_secret.clone(); let (tx, rx) = mpsc::channel::>(256); tokio::spawn(async move { - if let Err(err) = stream_exec_over_ssh( + // Wait for the supervisor's reverse CONNECT to deliver the relay stream. + let relay_stream = match tokio::time::timeout(std::time::Duration::from_secs(10), relay_rx) + .await + { + Ok(Ok(stream)) => stream, + Ok(Err(_)) => { + warn!(sandbox_id = %sandbox_id, channel_id = %channel_id, "ExecSandbox: relay channel dropped"); + let _ = tx + .send(Err(Status::unavailable("relay channel dropped"))) + .await; + return; + } + Err(_) => { + warn!(sandbox_id = %sandbox_id, channel_id = %channel_id, "ExecSandbox: relay open timed out"); + let _ = tx + .send(Err(Status::deadline_exceeded("relay open timed out"))) + .await; + return; + } + }; + + if let Err(err) = stream_exec_over_relay( tx.clone(), &sandbox_id, - &target_host, - target_port, + &channel_id, + relay_stream, &command_str, stdin_payload, timeout_seconds, request_tty, - &handshake_secret, ) .await { @@ -584,16 +610,6 @@ fn resolve_gateway(config: &openshell_core::Config) -> (String, u16) { (host, port) } -async fn resolve_sandbox_exec_target( - state: &ServerState, - sandbox: &Sandbox, -) -> Result<(String, u16), Status> { - match state.compute.resolve_sandbox_endpoint(sandbox).await? { - crate::compute::ResolvedEndpoint::Ip(ip, port) => Ok((ip.to_string(), port)), - crate::compute::ResolvedEndpoint::Host(host, port) => Ok((host, port)), - } -} - /// Shell-escape a value for embedding in a POSIX shell command. /// /// Wraps unsafe values in single quotes with the standard `'\''` idiom for @@ -646,133 +662,72 @@ fn build_remote_exec_command(req: &ExecSandboxRequest) -> Result Ok(result) } -/// Maximum number of attempts when establishing the SSH transport to a sandbox. -const SSH_CONNECT_MAX_ATTEMPTS: u32 = 6; - -/// Initial backoff duration between SSH connection retries. -const SSH_CONNECT_INITIAL_BACKOFF: std::time::Duration = std::time::Duration::from_millis(250); - -/// Maximum backoff duration between SSH connection retries. -const SSH_CONNECT_MAX_BACKOFF: std::time::Duration = std::time::Duration::from_secs(2); - -/// Returns `true` if the gRPC status represents a transient SSH connection error. -fn is_retryable_ssh_error(status: &Status) -> bool { - if status.code() != tonic::Code::Internal { - return false; - } - let msg = status.message(); - msg.contains("Connection reset by peer") - || msg.contains("Connection refused") - || msg.contains("failed to establish ssh transport") - || msg.contains("failed to connect to ssh proxy") - || msg.contains("failed to start ssh proxy") -} - +/// Execute a command over an SSH transport relayed through a supervisor session. +/// +/// This is the relay equivalent of `stream_exec_over_ssh`. Instead of dialing a +/// sandbox endpoint directly, the SSH transport runs over a `DuplexStream` that +/// is bridged to the supervisor's local SSH daemon via a reverse HTTP CONNECT +/// tunnel. #[allow(clippy::too_many_arguments)] -async fn stream_exec_over_ssh( +async fn stream_exec_over_relay( tx: mpsc::Sender>, sandbox_id: &str, - target_host: &str, - target_port: u16, + channel_id: &str, + relay_stream: tokio::io::DuplexStream, command: &str, stdin_payload: Vec, timeout_seconds: u32, request_tty: bool, - handshake_secret: &str, ) -> Result<(), Status> { let command_preview: String = command.chars().take(120).collect(); info!( sandbox_id = %sandbox_id, - target_host = %target_host, - target_port, + channel_id = %channel_id, command_len = command.len(), stdin_len = stdin_payload.len(), command_preview = %command_preview, - "ExecSandbox command started" + "ExecSandbox (relay): command started" ); - let (exit_code, proxy_task) = { - let mut last_err: Option = None; - - let mut result = None; - for attempt in 0..SSH_CONNECT_MAX_ATTEMPTS { - if attempt > 0 { - let backoff = (SSH_CONNECT_INITIAL_BACKOFF * 2u32.pow(attempt - 1)) - .min(SSH_CONNECT_MAX_BACKOFF); - warn!( - sandbox_id = %sandbox_id, - attempt = attempt + 1, - backoff_ms = %backoff.as_millis(), - error = %last_err.as_ref().unwrap(), - "Retrying SSH transport establishment" - ); - tokio::time::sleep(backoff).await; - } + let (local_proxy_port, proxy_task) = start_single_use_ssh_proxy_over_relay(relay_stream) + .await + .map_err(|e| Status::internal(format!("failed to start relay proxy: {e}")))?; + + let exec = run_exec_with_russh( + local_proxy_port, + command, + stdin_payload, + request_tty, + tx.clone(), + ); - let (local_proxy_port, proxy_task) = match start_single_use_ssh_proxy( - target_host, - target_port, - handshake_secret, - ) - .await - { - Ok(v) => v, - Err(e) => { - last_err = Some(Status::internal(format!("failed to start ssh proxy: {e}"))); - continue; - } - }; - - let exec = run_exec_with_russh( - local_proxy_port, - command, - stdin_payload.clone(), - request_tty, - tx.clone(), - ); + let exec_result = if timeout_seconds == 0 { + exec.await + } else if let Ok(r) = tokio::time::timeout( + std::time::Duration::from_secs(u64::from(timeout_seconds)), + exec, + ) + .await + { + r + } else { + let _ = tx + .send(Ok(ExecSandboxEvent { + payload: Some(openshell_core::proto::exec_sandbox_event::Payload::Exit( + ExecSandboxExit { exit_code: 124 }, + )), + })) + .await; + let _ = proxy_task.await; + return Ok(()); + }; - let exec_result = if timeout_seconds == 0 { - exec.await - } else if let Ok(r) = tokio::time::timeout( - std::time::Duration::from_secs(u64::from(timeout_seconds)), - exec, - ) - .await - { - r - } else { - let _ = tx - .send(Ok(ExecSandboxEvent { - payload: Some(openshell_core::proto::exec_sandbox_event::Payload::Exit( - ExecSandboxExit { exit_code: 124 }, - )), - })) - .await; - let _ = proxy_task.await; - return Ok(()); - }; - - match exec_result { - Ok(exit_code) => { - result = Some((exit_code, proxy_task)); - break; - } - Err(status) => { - let _ = proxy_task.await; - if is_retryable_ssh_error(&status) && attempt + 1 < SSH_CONNECT_MAX_ATTEMPTS { - last_err = Some(status); - continue; - } - return Err(status); - } - } + let exit_code = match exec_result { + Ok(code) => code, + Err(status) => { + let _ = proxy_task.await; + return Err(status); } - - result.ok_or_else(|| { - last_err.unwrap_or_else(|| { - Status::internal("ssh connection failed after exhausting retries") - }) - })? }; let _ = proxy_task.await; @@ -788,6 +743,28 @@ async fn stream_exec_over_ssh( Ok(()) } +/// Create a localhost SSH proxy that bridges to a relay DuplexStream. +/// +/// The proxy forwards raw SSH bytes between the `russh` client and the relay. +/// The supervisor bridges the relay to its Unix-socket SSH daemon; filesystem +/// permissions on that socket are the only access-control boundary. +async fn start_single_use_ssh_proxy_over_relay( + mut relay_stream: tokio::io::DuplexStream, +) -> Result<(u16, tokio::task::JoinHandle<()>), Box> { + let listener = TcpListener::bind(("127.0.0.1", 0)).await?; + let port = listener.local_addr()?.port(); + + let task = tokio::spawn(async move { + let Ok((mut client_conn, _)) = listener.accept().await else { + warn!("SSH relay proxy: failed to accept local connection"); + return; + }; + let _ = tokio::io::copy_bidirectional(&mut client_conn, &mut relay_stream).await; + }); + + Ok((port, task)) +} + #[derive(Debug, Clone, Copy)] struct SandboxSshClientHandler; @@ -914,158 +891,6 @@ async fn run_exec_with_russh( Ok(exit_code.unwrap_or(1)) } -/// Check whether an IP address is safe to use as an SSH proxy target. -fn is_safe_ssh_proxy_target(ip: std::net::IpAddr) -> bool { - match ip { - std::net::IpAddr::V4(v4) => !v4.is_loopback() && !v4.is_link_local(), - std::net::IpAddr::V6(v6) => { - if v6.is_loopback() { - return false; - } - if let Some(v4) = v6.to_ipv4_mapped() { - return !v4.is_loopback() && !v4.is_link_local(); - } - true - } - } -} - -fn is_explicit_loopback_exec_target(host: &str) -> bool { - if host.eq_ignore_ascii_case("localhost") { - return true; - } - - host.parse::() - .is_ok_and(|ip| ip.is_loopback()) -} - -async fn start_single_use_ssh_proxy( - target_host: &str, - target_port: u16, - handshake_secret: &str, -) -> Result<(u16, tokio::task::JoinHandle<()>), Box> { - let listener = TcpListener::bind(("127.0.0.1", 0)).await?; - let port = listener.local_addr()?.port(); - let target_host = target_host.to_string(); - let allow_explicit_loopback = is_explicit_loopback_exec_target(target_host.as_str()); - let handshake_secret = handshake_secret.to_string(); - - let task = tokio::spawn(async move { - let Ok((mut client_conn, _)) = listener.accept().await else { - warn!("SSH proxy: failed to accept local connection"); - return; - }; - - let addr_str = format!("{target_host}:{target_port}"); - let resolved = match tokio::net::lookup_host(&addr_str).await { - Ok(mut addrs) => { - if let Some(addr) = addrs.next() { - addr - } else { - warn!(target_host = %target_host, "SSH proxy: DNS resolution returned no addresses"); - return; - } - } - Err(e) => { - warn!(target_host = %target_host, error = %e, "SSH proxy: DNS resolution failed"); - return; - } - }; - - if !allow_explicit_loopback && !is_safe_ssh_proxy_target(resolved.ip()) { - warn!( - target_host = %target_host, - resolved_ip = %resolved.ip(), - "SSH proxy: target resolved to blocked IP range (loopback or link-local)" - ); - return; - } - - debug!( - target_host = %target_host, - resolved_ip = %resolved.ip(), - target_port, - "SSH proxy: connecting to validated target" - ); - - let Ok(mut sandbox_conn) = TcpStream::connect(resolved).await else { - warn!(target_host = %target_host, resolved_ip = %resolved.ip(), target_port, "SSH proxy: failed to connect to sandbox"); - return; - }; - let Ok(preface) = build_preface(&uuid::Uuid::new_v4().to_string(), &handshake_secret) - else { - warn!("SSH proxy: failed to build handshake preface"); - return; - }; - if let Err(e) = sandbox_conn.write_all(preface.as_bytes()).await { - warn!(error = %e, "SSH proxy: failed to send handshake preface"); - return; - } - let mut response = String::new(); - if let Err(e) = read_line(&mut sandbox_conn, &mut response).await { - warn!(error = %e, "SSH proxy: failed to read handshake response"); - return; - } - if response.trim() != "OK" { - warn!(response = %response.trim(), "SSH proxy: handshake rejected by sandbox"); - return; - } - let _ = tokio::io::copy_bidirectional(&mut client_conn, &mut sandbox_conn).await; - }); - - Ok((port, task)) -} - -fn build_preface( - token: &str, - secret: &str, -) -> Result> { - let timestamp = i64::try_from( - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_err(|_| "time error")? - .as_secs(), - ) - .map_err(|_| "time error")?; - let nonce = uuid::Uuid::new_v4().to_string(); - let payload = format!("{token}|{timestamp}|{nonce}"); - let signature = hmac_sha256(secret.as_bytes(), payload.as_bytes()); - Ok(format!("NSSH1 {token} {timestamp} {nonce} {signature}\n")) -} - -async fn read_line( - stream: &mut TcpStream, - buf: &mut String, -) -> Result<(), Box> { - let mut bytes = Vec::new(); - loop { - let mut byte = [0_u8; 1]; - let n = stream.read(&mut byte).await?; - if n == 0 { - break; - } - if byte[0] == b'\n' { - break; - } - bytes.push(byte[0]); - if bytes.len() > 1024 { - break; - } - } - *buf = String::from_utf8_lossy(&bytes).to_string(); - Ok(()) -} - -fn hmac_sha256(key: &[u8], data: &[u8]) -> String { - use hmac::{Hmac, Mac}; - use sha2::Sha256; - - let mut mac = Hmac::::new_from_slice(key).expect("hmac key"); - mac.update(data); - let result = mac.finalize().into_bytes(); - hex::encode(result) -} - // --------------------------------------------------------------------------- // Tests // --------------------------------------------------------------------------- @@ -1171,74 +996,6 @@ mod tests { assert!(build_remote_exec_command(&req).is_err()); } - // ---- is_safe_ssh_proxy_target ---- - - #[test] - fn ssh_proxy_target_allows_pod_network_ips() { - use std::net::{IpAddr, Ipv4Addr}; - assert!(is_safe_ssh_proxy_target(IpAddr::V4(Ipv4Addr::new( - 10, 0, 0, 5 - )))); - assert!(is_safe_ssh_proxy_target(IpAddr::V4(Ipv4Addr::new( - 172, 16, 0, 1 - )))); - assert!(is_safe_ssh_proxy_target(IpAddr::V4(Ipv4Addr::new( - 192, 168, 1, 100 - )))); - } - - #[test] - fn ssh_proxy_target_blocks_loopback() { - use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; - assert!(!is_safe_ssh_proxy_target(IpAddr::V4(Ipv4Addr::new( - 127, 0, 0, 1 - )))); - assert!(!is_safe_ssh_proxy_target(IpAddr::V4(Ipv4Addr::new( - 127, 0, 0, 2 - )))); - assert!(!is_safe_ssh_proxy_target(IpAddr::V6(Ipv6Addr::LOCALHOST))); - } - - #[test] - fn ssh_proxy_target_blocks_link_local() { - use std::net::{IpAddr, Ipv4Addr}; - assert!(!is_safe_ssh_proxy_target(IpAddr::V4(Ipv4Addr::new( - 169, 254, 169, 254 - )))); - assert!(!is_safe_ssh_proxy_target(IpAddr::V4(Ipv4Addr::new( - 169, 254, 0, 1 - )))); - } - - #[test] - fn ssh_proxy_target_blocks_ipv4_mapped_ipv6_loopback() { - use std::net::IpAddr; - let ip: IpAddr = "::ffff:127.0.0.1".parse().unwrap(); - assert!(!is_safe_ssh_proxy_target(ip)); - } - - #[test] - fn ssh_proxy_target_blocks_ipv4_mapped_ipv6_link_local() { - use std::net::IpAddr; - let ip: IpAddr = "::ffff:169.254.169.254".parse().unwrap(); - assert!(!is_safe_ssh_proxy_target(ip)); - } - - #[test] - fn explicit_loopback_exec_target_is_allowed() { - assert!(is_explicit_loopback_exec_target("127.0.0.1")); - assert!(is_explicit_loopback_exec_target("::1")); - assert!(is_explicit_loopback_exec_target("localhost")); - } - - #[test] - fn non_loopback_exec_target_is_not_treated_as_explicit_loopback() { - assert!(!is_explicit_loopback_exec_target("10.0.0.5")); - assert!(!is_explicit_loopback_exec_target( - "sandbox.default.svc.cluster.local" - )); - } - // ---- petname / generate_name ---- #[test] diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index da56a5c940..0894e53422 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -30,6 +30,7 @@ mod persistence; mod sandbox_index; mod sandbox_watch; mod ssh_tunnel; +pub mod supervisor_session; mod tls; pub mod tracing_bus; mod ws_tunnel; @@ -85,6 +86,9 @@ pub struct ServerState { /// set/delete operation, including the precedence check on sandbox /// mutations that reads global state. pub settings_mutex: tokio::sync::Mutex<()>, + + /// Registry of active supervisor sessions and pending relay channels. + pub supervisor_sessions: supervisor_session::SupervisorSessionRegistry, } fn is_benign_tls_handshake_failure(error: &std::io::Error) -> bool { @@ -115,6 +119,7 @@ impl ServerState { ssh_connections_by_token: Mutex::new(HashMap::new()), ssh_connections_by_sandbox: Mutex::new(HashMap::new()), settings_mutex: tokio::sync::Mutex::new(()), + supervisor_sessions: supervisor_session::SupervisorSessionRegistry::new(), } } } @@ -165,6 +170,7 @@ pub async fn run_server( state.compute.spawn_watchers(); ssh_tunnel::spawn_session_reaper(store.clone(), Duration::from_secs(3600)); + supervisor_session::spawn_relay_reaper(state.clone(), Duration::from_secs(30)); // Create the multiplexed service let service = MultiplexService::new(state.clone()); @@ -247,8 +253,13 @@ async fn build_compute_runtime( default_image: config.sandbox_image.clone(), image_pull_policy: config.sandbox_image_pull_policy.clone(), grpc_endpoint: config.grpc_endpoint.clone(), - ssh_listen_addr: format!("0.0.0.0:{}", config.sandbox_ssh_port), - ssh_port: config.sandbox_ssh_port, + // Filesystem path to the supervisor's Unix-socket SSH daemon. + // The path lives in a root-only directory so only the + // supervisor can connect; the gateway reaches it through the + // RelayStream bridge, not directly. Override via + // `sandbox_ssh_socket_path` in the config for deployments + // where multiple supervisors share a filesystem. + ssh_socket_path: config.sandbox_ssh_socket_path.clone(), ssh_handshake_secret: config.ssh_handshake_secret.clone(), ssh_handshake_skew_secs: config.ssh_handshake_skew_secs, client_tls_secret_name: config.client_tls_secret_name.clone(), diff --git a/crates/openshell-server/src/multiplex.rs b/crates/openshell-server/src/multiplex.rs index 5ba44b1ec0..0f6b9449fd 100644 --- a/crates/openshell-server/src/multiplex.rs +++ b/crates/openshell-server/src/multiplex.rs @@ -62,7 +62,17 @@ impl MultiplexService { let service = MultiplexedService::new(grpc_service, http_service); - Builder::new(TokioExecutor::new()) + // HTTP/2 adaptive flow control. Default windows (64 KiB / 64 KiB) + // throttle the RelayStream data plane to ~500 Mbps on LAN. Instead + // of committing to a fixed large window (which worst-case pins + // `max_concurrent_streams × stream_window` bytes per connection), + // we let hyper/h2 auto-size based on the measured bandwidth-delay + // product. Idle streams stay tiny; busy bulk streams grow as + // needed. Overrides any fixed initial_*_window_size settings. + let mut builder = Builder::new(TokioExecutor::new()); + builder.http2().adaptive_window(true); + + builder .serve_connection_with_upgrades(TokioIo::new(stream), service) .await?; diff --git a/crates/openshell-server/src/ssh_tunnel.rs b/crates/openshell-server/src/ssh_tunnel.rs index 536513ccd8..b635efd88c 100644 --- a/crates/openshell-server/src/ssh_tunnel.rs +++ b/crates/openshell-server/src/ssh_tunnel.rs @@ -6,24 +6,19 @@ use axum::{Router, extract::State, http::Method, response::IntoResponse, routing::any}; use http::StatusCode; use hyper::Request; -use hyper::upgrade::Upgraded; use hyper_util::rt::TokioIo; use openshell_core::proto::{Sandbox, SandboxPhase, SshSession}; use prost::Message; -use std::net::SocketAddr; use std::sync::Arc; use std::time::Duration; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::net::TcpStream; +use tokio::io::AsyncWriteExt; use tracing::{info, warn}; -use uuid::Uuid; use crate::ServerState; use crate::persistence::{ObjectId, ObjectName, ObjectType, Store}; const HEADER_SANDBOX_ID: &str = "x-sandbox-id"; const HEADER_TOKEN: &str = "x-sandbox-token"; -const PREFACE_MAGIC: &str = "NSSH1"; /// Maximum concurrent SSH tunnel connections per session token. const MAX_CONNECTIONS_PER_TOKEN: u32 = 3; @@ -40,6 +35,39 @@ fn redact_token(token: &str) -> String { /// Maximum concurrent SSH tunnel connections per sandbox. const MAX_CONNECTIONS_PER_SANDBOX: u32 = 20; +fn acquire_connection_slots( + token_counts: &std::sync::Mutex>, + sandbox_counts: &std::sync::Mutex>, + token: &str, + sandbox_id: &str, +) -> Result<(), ConnectionLimit> { + { + let mut counts = token_counts.lock().unwrap(); + let count = counts.entry(token.to_string()).or_insert(0); + if *count >= MAX_CONNECTIONS_PER_TOKEN { + return Err(ConnectionLimit::PerToken); + } + *count += 1; + } + + { + let mut counts = sandbox_counts.lock().unwrap(); + let count = counts.entry(sandbox_id.to_string()).or_insert(0); + if *count >= MAX_CONNECTIONS_PER_SANDBOX { + decrement_connection_count(token_counts, token); + return Err(ConnectionLimit::PerSandbox); + } + *count += 1; + } + + Ok(()) +} + +enum ConnectionLimit { + PerToken, + PerSandbox, +} + pub fn router(state: Arc) -> Router { Router::new() .route("/connect/ssh", any(ssh_connect)) @@ -100,69 +128,82 @@ async fn ssh_connect( return StatusCode::PRECONDITION_FAILED.into_response(); } - let connect_target = match state.compute.resolve_sandbox_endpoint(&sandbox).await { - Ok(crate::compute::ResolvedEndpoint::Ip(ip, port)) => { - ConnectTarget::Ip(SocketAddr::new(ip, port)) - } - Ok(crate::compute::ResolvedEndpoint::Host(host, port)) => ConnectTarget::Host(host, port), - Err(status) if status.code() == tonic::Code::FailedPrecondition => { - return StatusCode::PRECONDITION_FAILED.into_response(); - } - Err(err) => { - warn!(error = %err, "Failed to resolve sandbox endpoint"); - return StatusCode::BAD_GATEWAY.into_response(); - } - }; - // Enforce per-token concurrent connection limit. - { - let mut counts = state.ssh_connections_by_token.lock().unwrap(); - let count = counts.entry(token.clone()).or_insert(0); - if *count >= MAX_CONNECTIONS_PER_TOKEN { - warn!(token = %redact_token(&token), "SSH tunnel: per-token connection limit reached"); - return StatusCode::TOO_MANY_REQUESTS.into_response(); + // Enforce connection caps *before* opening a relay — otherwise denied + // calls churn pending relay slots and wake the supervisor until the relay + // timeout elapses. + if let Err(limit) = acquire_connection_slots( + &state.ssh_connections_by_token, + &state.ssh_connections_by_sandbox, + &token, + &sandbox_id, + ) { + match limit { + ConnectionLimit::PerToken => { + warn!(token = %redact_token(&token), "SSH tunnel: per-token connection limit reached"); + } + ConnectionLimit::PerSandbox => { + warn!(sandbox_id = %sandbox_id, "SSH tunnel: per-sandbox connection limit reached"); + } } - *count += 1; + return StatusCode::TOO_MANY_REQUESTS.into_response(); } - // Enforce per-sandbox concurrent connection limit. + // Open a relay channel through the supervisor session. Use a generous + // 30s session-wait timeout because `/connect/ssh` is typically called + // immediately after `sandbox create`, so we need to cover the supervisor's + // initial TLS + gRPC handshake on a cold-started pod. The old + // direct-connect path tolerated ~34s here for similar reasons. + let (channel_id, relay_rx) = match state + .supervisor_sessions + .open_relay(&sandbox_id, Duration::from_secs(30)) + .await { - let mut counts = state.ssh_connections_by_sandbox.lock().unwrap(); - let count = counts.entry(sandbox_id.clone()).or_insert(0); - if *count >= MAX_CONNECTIONS_PER_SANDBOX { - // Roll back the per-token increment. - let mut token_counts = state.ssh_connections_by_token.lock().unwrap(); - if let Some(c) = token_counts.get_mut(&token) { - *c = c.saturating_sub(1); - if *c == 0 { - token_counts.remove(&token); - } - } - warn!(sandbox_id = %sandbox_id, "SSH tunnel: per-sandbox connection limit reached"); - return StatusCode::TOO_MANY_REQUESTS.into_response(); + Ok(pair) => pair, + Err(status) => { + warn!(sandbox_id = %sandbox_id, error = %status.message(), "SSH tunnel: supervisor session not available"); + decrement_connection_count(&state.ssh_connections_by_token, &token); + decrement_connection_count(&state.ssh_connections_by_sandbox, &sandbox_id); + return StatusCode::BAD_GATEWAY.into_response(); } - *count += 1; - } + }; - let handshake_secret = state.config.ssh_handshake_secret.clone(); let sandbox_id_clone = sandbox_id.clone(); let token_clone = token.clone(); let state_clone = state.clone(); let upgrade = hyper::upgrade::on(req); tokio::spawn(async move { - match upgrade.await { - Ok(mut upgraded) => { - if let Err(err) = handle_tunnel( - &mut upgraded, - connect_target, - &token_clone, - &handshake_secret, + // Wait for the supervisor to open its `RelayStream` and deliver the + // bridge half of the relay. + let mut relay = match tokio::time::timeout(Duration::from_secs(10), relay_rx).await { + Ok(Ok(stream)) => stream, + Ok(Err(_)) => { + warn!(sandbox_id = %sandbox_id_clone, channel_id = %channel_id, "SSH tunnel: relay channel dropped"); + decrement_connection_count(&state_clone.ssh_connections_by_token, &token_clone); + decrement_connection_count( + &state_clone.ssh_connections_by_sandbox, &sandbox_id_clone, - ) - .await - { - warn!(error = %err, "SSH tunnel failure"); - } + ); + return; + } + Err(_) => { + warn!(sandbox_id = %sandbox_id_clone, channel_id = %channel_id, "SSH tunnel: relay open timed out"); + decrement_connection_count(&state_clone.ssh_connections_by_token, &token_clone); + decrement_connection_count( + &state_clone.ssh_connections_by_sandbox, + &sandbox_id_clone, + ); + return; + } + }; + + info!(sandbox_id = %sandbox_id_clone, channel_id = %channel_id, "SSH tunnel: relay established, bridging client"); + + match upgrade.await { + Ok(upgraded) => { + let mut upgraded = TokioIo::new(upgraded); + let _ = tokio::io::copy_bidirectional(&mut upgraded, &mut relay).await; + let _ = AsyncWriteExt::shutdown(&mut upgraded).await; } Err(err) => { warn!(error = %err, "SSH upgrade failed"); @@ -177,90 +218,6 @@ async fn ssh_connect( StatusCode::OK.into_response() } -async fn handle_tunnel( - upgraded: &mut Upgraded, - target: ConnectTarget, - token: &str, - secret: &str, - sandbox_id: &str, -) -> Result<(), Box> { - // The sandbox pod may not be network-reachable immediately after the CRD - // reports Ready (DNS propagation, pod IP assignment, SSH server startup). - // Retry the TCP connection with exponential backoff. - let mut upstream = None; - let mut last_err = None; - let delays = [ - Duration::from_millis(100), - Duration::from_millis(250), - Duration::from_millis(500), - Duration::from_secs(1), - Duration::from_secs(2), - Duration::from_secs(5), - Duration::from_secs(10), - Duration::from_secs(15), - ]; - let target_desc = match &target { - ConnectTarget::Ip(addr) => format!("{addr}"), - ConnectTarget::Host(host, port) => format!("{host}:{port}"), - }; - info!(sandbox_id = %sandbox_id, target = %target_desc, "SSH tunnel: connecting to sandbox"); - for (attempt, delay) in std::iter::once(&Duration::ZERO) - .chain(delays.iter()) - .enumerate() - { - if !delay.is_zero() { - info!(sandbox_id = %sandbox_id, attempt = attempt + 1, delay_ms = delay.as_millis() as u64, "SSH tunnel: retrying TCP connect"); - tokio::time::sleep(*delay).await; - } - let result = match &target { - ConnectTarget::Ip(addr) => TcpStream::connect(addr).await, - ConnectTarget::Host(host, port) => TcpStream::connect((host.as_str(), *port)).await, - }; - match result { - Ok(stream) => { - info!( - sandbox_id = %sandbox_id, - attempts = attempt + 1, - "SSH tunnel: TCP connected to sandbox" - ); - upstream = Some(stream); - break; - } - Err(err) => { - info!(sandbox_id = %sandbox_id, attempt = attempt + 1, error = %err, "SSH tunnel: TCP connect failed"); - last_err = Some(err); - } - } - } - let mut upstream = upstream.ok_or_else(|| { - let err = last_err.unwrap(); - format!("failed to connect to sandbox after retries: {err}") - })?; - upstream.set_nodelay(true)?; - info!(sandbox_id = %sandbox_id, "SSH tunnel: sending NSSH1 handshake preface"); - let preface = build_preface(token, secret)?; - upstream.write_all(preface.as_bytes()).await?; - - info!(sandbox_id = %sandbox_id, "SSH tunnel: waiting for handshake response"); - let mut response = String::new(); - read_line(&mut upstream, &mut response).await?; - info!(sandbox_id = %sandbox_id, response = %response.trim(), "SSH tunnel: handshake response received"); - if response.trim() != "OK" { - return Err("sandbox handshake rejected".into()); - } - - info!(sandbox_id = %sandbox_id, "SSH tunnel established"); - let mut upgraded = TokioIo::new(upgraded); - // Discard the result entirely – connection-close errors are expected when - // the SSH session ends and do not represent a failure worth propagating. - let _ = tokio::io::copy_bidirectional(&mut upgraded, &mut upstream).await; - // Gracefully shut down the write-half of the upgraded connection so the - // client receives a clean EOF instead of a TCP RST. This gives SSH time - // to read any remaining protocol data (e.g. exit-status) from its buffer. - let _ = AsyncWriteExt::shutdown(&mut upgraded).await; - Ok(()) -} - fn header_value(headers: &http::HeaderMap, name: &str) -> Result { let value = headers .get(name) @@ -275,58 +232,6 @@ fn header_value(headers: &http::HeaderMap, name: &str) -> Result Result> { - let timestamp = i64::try_from( - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_err(|_| "time error")? - .as_secs(), - ) - .map_err(|_| "time error")?; - let nonce = Uuid::new_v4().to_string(); - let payload = format!("{token}|{timestamp}|{nonce}"); - let signature = hmac_sha256(secret.as_bytes(), payload.as_bytes()); - Ok(format!( - "{PREFACE_MAGIC} {token} {timestamp} {nonce} {signature}\n" - )) -} - -async fn read_line( - stream: &mut TcpStream, - buf: &mut String, -) -> Result<(), Box> { - let mut bytes = Vec::new(); - loop { - let mut byte = [0u8; 1]; - let n = stream.read(&mut byte).await?; - if n == 0 { - break; - } - if byte[0] == b'\n' { - break; - } - bytes.push(byte[0]); - if bytes.len() > 1024 { - break; - } - } - *buf = String::from_utf8_lossy(&bytes).to_string(); - Ok(()) -} - -fn hmac_sha256(key: &[u8], data: &[u8]) -> String { - use hmac::{Hmac, Mac}; - use sha2::Sha256; - - let mut mac = Hmac::::new_from_slice(key).expect("hmac key"); - mac.update(data); - let result = mac.finalize().into_bytes(); - hex::encode(result) -} - impl ObjectType for SshSession { fn object_type() -> &'static str { "ssh_session" @@ -345,11 +250,6 @@ impl ObjectName for SshSession { } } -enum ConnectTarget { - Ip(SocketAddr), - Host(String, u16), -} - /// Decrement a connection count entry, removing it if it reaches zero. fn decrement_connection_count( counts: &std::sync::Mutex>, @@ -491,6 +391,36 @@ mod tests { assert!(current >= MAX_CONNECTIONS_PER_SANDBOX); } + #[test] + fn acquire_connection_slots_rejects_per_token_limit_without_touching_sandbox() { + let token_counts: Mutex> = Mutex::new(HashMap::new()); + let sandbox_counts: Mutex> = Mutex::new(HashMap::new()); + token_counts + .lock() + .unwrap() + .insert("tok1".to_string(), MAX_CONNECTIONS_PER_TOKEN); + + let result = acquire_connection_slots(&token_counts, &sandbox_counts, "tok1", "sbx1"); + + assert!(matches!(result, Err(ConnectionLimit::PerToken))); + assert!(sandbox_counts.lock().unwrap().is_empty()); + } + + #[test] + fn acquire_connection_slots_rolls_back_token_increment_on_sandbox_limit() { + let token_counts: Mutex> = Mutex::new(HashMap::new()); + let sandbox_counts: Mutex> = Mutex::new(HashMap::new()); + sandbox_counts + .lock() + .unwrap() + .insert("sbx1".to_string(), MAX_CONNECTIONS_PER_SANDBOX); + + let result = acquire_connection_slots(&token_counts, &sandbox_counts, "tok1", "sbx1"); + + assert!(matches!(result, Err(ConnectionLimit::PerSandbox))); + assert!(token_counts.lock().unwrap().is_empty()); + } + // ---- Session reaper tests ---- #[tokio::test] diff --git a/crates/openshell-server/src/supervisor_session.rs b/crates/openshell-server/src/supervisor_session.rs new file mode 100644 index 0000000000..f81ee9e3cb --- /dev/null +++ b/crates/openshell-server/src/supervisor_session.rs @@ -0,0 +1,1262 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::HashMap; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use tokio::sync::{mpsc, oneshot}; +use tokio_stream::wrappers::ReceiverStream; +use tonic::{Request, Response, Status}; +use tracing::{info, warn}; +use uuid::Uuid; + +use openshell_core::proto::{ + GatewayMessage, RelayFrame, RelayInit, RelayOpen, Sandbox, SessionAccepted, SupervisorMessage, + gateway_message, supervisor_message, +}; + +use crate::ServerState; + +const HEARTBEAT_INTERVAL_SECS: u32 = 15; +const RELAY_PENDING_TIMEOUT: Duration = Duration::from_secs(10); +/// Initial backoff between session-availability polls in `wait_for_session`. +const SESSION_WAIT_INITIAL_BACKOFF: Duration = Duration::from_millis(100); +/// Maximum backoff between session-availability polls in `wait_for_session`. +const SESSION_WAIT_MAX_BACKOFF: Duration = Duration::from_secs(2); +/// Upper bound on unclaimed relay channels across all sandboxes. Caps the +/// memory a misbehaving caller can pin by calling `open_relay` repeatedly +/// while the supervisor never claims (or isn't responding). Sized generously +/// so normal bursts pass through; exceeding it returns `ResourceExhausted`. +const MAX_PENDING_RELAYS: usize = 256; +/// Upper bound on concurrent unclaimed relay channels for a single sandbox. +/// Enforces the same shape per sandbox so one misbehaving sandbox can't +/// consume the entire global budget. Sits above the SSH-tunnel per-sandbox +/// cap (20) so tunnel-specific limits still fire first for that caller. +const MAX_PENDING_RELAYS_PER_SANDBOX: usize = 32; + +// --------------------------------------------------------------------------- +// Session registry +// --------------------------------------------------------------------------- + +/// A live supervisor session handle. +struct LiveSession { + #[allow(dead_code)] + sandbox_id: String, + /// Uniquely identifies this session instance. Used by cleanup to avoid + /// removing a session that has since been superseded by a reconnect. + session_id: String, + tx: mpsc::Sender, + /// Fires when this session is superseded by a reconnect so the old session + /// task can exit promptly — dropping its own `tx` clone and closing the + /// outbound stream. Without this, a concurrent `open_relay` that grabbed + /// the old session's `tx` just before supersede could still enqueue a + /// `RelayOpen` onto the stale stream and sit until the relay timeout. + shutdown: oneshot::Sender<()>, + #[allow(dead_code)] + connected_at: Instant, +} + +/// Holds a oneshot sender that will deliver the upgraded relay stream. +type RelayStreamSender = oneshot::Sender; + +/// Registry of active supervisor sessions and pending relay channels. +#[derive(Default)] +pub struct SupervisorSessionRegistry { + /// sandbox_id -> live session handle. + sessions: Mutex>, + /// channel_id -> oneshot sender for the reverse CONNECT stream. + pending_relays: Mutex>, +} + +struct PendingRelay { + sender: RelayStreamSender, + sandbox_id: String, + created_at: Instant, +} + +impl std::fmt::Debug for SupervisorSessionRegistry { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let session_count = self.sessions.lock().unwrap().len(); + let pending_count = self.pending_relays.lock().unwrap().len(); + f.debug_struct("SupervisorSessionRegistry") + .field("sessions", &session_count) + .field("pending_relays", &pending_count) + .finish() + } +} + +impl SupervisorSessionRegistry { + pub fn new() -> Self { + Self::default() + } + + /// Register a live supervisor session for the given sandbox. + /// + /// If a previous session exists for the same sandbox, its shutdown signal + /// is fired so the old session task exits promptly. Returns `true` iff a + /// previous session was superseded. + pub fn register( + &self, + sandbox_id: String, + session_id: String, + tx: mpsc::Sender, + shutdown: oneshot::Sender<()>, + ) -> bool { + let mut sessions = self.sessions.lock().unwrap(); + let previous = sessions.remove(&sandbox_id); + sessions.insert( + sandbox_id.clone(), + LiveSession { + sandbox_id, + session_id, + tx, + shutdown, + connected_at: Instant::now(), + }, + ); + match previous { + Some(prev) => { + // Best-effort — the old task may have already exited. + let _ = prev.shutdown.send(()); + true + } + None => false, + } + } + + /// Remove the session for a sandbox. + fn remove(&self, sandbox_id: &str) { + self.sessions.lock().unwrap().remove(sandbox_id); + } + + /// Remove the session only if its `session_id` matches the one we are + /// cleaning up. Returns `true` if the entry was removed. + /// + /// This guards against the supersede race: an old session's task may + /// finish long after a new session has taken its place. The old task's + /// cleanup must not evict the new registration. + fn remove_if_current(&self, sandbox_id: &str, session_id: &str) -> bool { + let mut sessions = self.sessions.lock().unwrap(); + let is_current = sessions + .get(sandbox_id) + .is_some_and(|s| s.session_id == session_id); + if is_current { + sessions.remove(sandbox_id); + } + is_current + } + + /// Look up the sender for a supervisor session, waiting up to `timeout` + /// for it to appear if absent. + /// + /// Uses exponential backoff (100ms → 2s) while polling the sessions map. + async fn wait_for_session( + &self, + sandbox_id: &str, + timeout: Duration, + ) -> Result, Status> { + let deadline = Instant::now() + timeout; + let mut backoff = SESSION_WAIT_INITIAL_BACKOFF; + + loop { + if let Some(tx) = self.lookup_session(sandbox_id) { + return Ok(tx); + } + if Instant::now() + backoff > deadline { + return Err(Status::unavailable("supervisor session not connected")); + } + tokio::time::sleep(backoff).await; + backoff = (backoff * 2).min(SESSION_WAIT_MAX_BACKOFF); + } + } + + fn lookup_session(&self, sandbox_id: &str) -> Option> { + self.sessions + .lock() + .unwrap() + .get(sandbox_id) + .map(|s| s.tx.clone()) + } + + fn pending_channel_ids(&self, sandbox_id: &str) -> Vec { + self.pending_relays + .lock() + .unwrap() + .iter() + .filter(|(_, pending)| pending.sandbox_id == sandbox_id) + .map(|(channel_id, _)| channel_id.clone()) + .collect() + } + + /// Open a relay channel and return a receiver for the supervisor-side + /// stream. + /// + /// Sends `RelayOpen` over the supervisor's gRPC session and returns a + /// oneshot receiver that resolves once the supervisor opens its reverse + /// HTTP CONNECT to `/relay/{channel_id}`. + /// + /// If the session is not currently registered, this method waits up to + /// `session_wait_timeout` for it to appear. A session may be temporarily + /// absent for several reasons — all of which look identical from here: + /// + /// - startup race: the sandbox just reported Ready but the supervisor's + /// `ConnectSupervisor` gRPC handshake hasn't completed yet + /// - transient disconnect: the session was up but got dropped (network + /// blip, gateway restart, supervisor restart) and the supervisor is + /// in its reconnect backoff loop + /// + /// Callers pick the timeout based on how much patience the caller needs. + /// A first `sandbox connect` right after `sandbox create` may need to + /// wait for the supervisor's initial TLS + gRPC handshake (tens of + /// seconds on a slow cluster), while mid-lifetime calls typically just + /// need to cover a short reconnect window. + pub async fn open_relay( + &self, + sandbox_id: &str, + session_wait_timeout: Duration, + ) -> Result<(String, oneshot::Receiver), Status> { + let tx = self + .wait_for_session(sandbox_id, session_wait_timeout) + .await?; + + let channel_id = Uuid::new_v4().to_string(); + + // Register the pending relay before sending RelayOpen to avoid a race. + // Both caps are checked and the insert happens under a single lock hold + // so two concurrent calls can't both observe "under the cap" and then + // both insert past it. + let (relay_tx, relay_rx) = oneshot::channel(); + { + let mut pending = self.pending_relays.lock().unwrap(); + if pending.len() >= MAX_PENDING_RELAYS { + return Err(Status::resource_exhausted(format!( + "gateway relay capacity reached ({MAX_PENDING_RELAYS} in flight)" + ))); + } + let per_sandbox = pending + .values() + .filter(|p| p.sandbox_id == sandbox_id) + .count(); + if per_sandbox >= MAX_PENDING_RELAYS_PER_SANDBOX { + return Err(Status::resource_exhausted(format!( + "per-sandbox relay limit reached ({MAX_PENDING_RELAYS_PER_SANDBOX} in flight for {sandbox_id})" + ))); + } + pending.insert( + channel_id.clone(), + PendingRelay { + sender: relay_tx, + sandbox_id: sandbox_id.to_string(), + created_at: Instant::now(), + }, + ); + } + + let msg = GatewayMessage { + payload: Some(gateway_message::Payload::RelayOpen(RelayOpen { + channel_id: channel_id.clone(), + })), + }; + + if tx.send(msg).await.is_err() { + // Session dropped between our lookup and send. + self.pending_relays.lock().unwrap().remove(&channel_id); + return Err(Status::unavailable("supervisor session disconnected")); + } + + Ok((channel_id, relay_rx)) + } + + /// Claim a pending relay channel. Called by the /relay/{channel_id} HTTP handler + /// when the supervisor's reverse CONNECT arrives. + /// + /// Returns the DuplexStream half that the supervisor side should read/write. + pub fn claim_relay(&self, channel_id: &str) -> Result { + let pending = { + let mut map = self.pending_relays.lock().unwrap(); + map.remove(channel_id) + .ok_or_else(|| Status::not_found("unknown or expired relay channel"))? + }; + + if pending.created_at.elapsed() > RELAY_PENDING_TIMEOUT { + return Err(Status::deadline_exceeded("relay channel timed out")); + } + + // Create a duplex stream pair: one end for the gateway bridge, one for + // the supervisor HTTP CONNECT handler. + let (gateway_stream, supervisor_stream) = tokio::io::duplex(64 * 1024); + + // Send the gateway-side stream to the waiter (ssh_tunnel or exec handler). + if pending.sender.send(gateway_stream).is_err() { + return Err(Status::internal("relay requester dropped")); + } + + Ok(supervisor_stream) + } + + /// Remove all pending relays that have exceeded the timeout. + pub fn reap_expired_relays(&self) { + let mut map = self.pending_relays.lock().unwrap(); + map.retain(|_, pending| pending.created_at.elapsed() <= RELAY_PENDING_TIMEOUT); + } + + /// Clean up all state for a sandbox (session + pending relays). + pub fn cleanup_sandbox(&self, sandbox_id: &str) { + self.remove(sandbox_id); + } + + pub async fn replay_pending_relays(&self, sandbox_id: &str, tx: &mpsc::Sender) { + for channel_id in self.pending_channel_ids(sandbox_id) { + let msg = GatewayMessage { + payload: Some(gateway_message::Payload::RelayOpen(RelayOpen { + channel_id: channel_id.clone(), + })), + }; + if tx.send(msg).await.is_err() { + warn!(sandbox_id = %sandbox_id, channel_id = %channel_id, "supervisor session: failed to replay pending relay to superseding session"); + break; + } + } + } +} + +/// Spawn a background task that periodically reaps expired pending relay +/// entries. +/// +/// Pending entries are normally consumed either when the supervisor opens its +/// reverse CONNECT (via `claim_relay`) or by the gateway-side waiter timing +/// out. If neither happens — e.g., the supervisor crashed after acknowledging +/// `RelayOpen` but before initiating `RelayStream` — the entry would otherwise +/// sit in the map indefinitely. This sweeper bounds that leak. +pub fn spawn_relay_reaper(state: Arc, interval: Duration) { + tokio::spawn(async move { + loop { + tokio::time::sleep(interval).await; + state.supervisor_sessions.reap_expired_relays(); + } + }); +} + +async fn require_persisted_sandbox( + store: &Arc, + sandbox_id: &str, +) -> Result<(), Status> { + let sandbox = store + .get_message::(sandbox_id) + .await + .map_err(|err| Status::internal(format!("failed to load sandbox: {err}")))?; + + if sandbox.is_none() { + return Err(Status::not_found("sandbox not found")); + } + + Ok(()) +} + +// --------------------------------------------------------------------------- +// RelayStream gRPC handler +// --------------------------------------------------------------------------- + +/// Size of chunks read from the gateway-side DuplexStream when forwarding +/// bytes back to the supervisor over the gRPC response stream. +const RELAY_STREAM_CHUNK_SIZE: usize = 16 * 1024; + +/// Handle a RelayStream RPC from a supervisor. The first inbound `RelayFrame` +/// must carry a `RelayInit` identifying the pending relay; subsequent frames +/// carry raw bytes forward to the gateway-side waiter. Bytes flowing the other +/// way are chunked and sent as `RelayFrame::data` messages back over the +/// response stream. +pub async fn handle_relay_stream( + registry: &SupervisorSessionRegistry, + request: Request>, +) -> Result< + Response< + Pin> + Send + 'static>>, + >, + Status, +> { + let mut inbound = request.into_inner(); + + // First frame must identify the channel. + let first = inbound + .message() + .await? + .ok_or_else(|| Status::invalid_argument("empty RelayStream"))?; + let channel_id = match first.payload { + Some(openshell_core::proto::relay_frame::Payload::Init(RelayInit { channel_id })) + if !channel_id.is_empty() => + { + channel_id + } + _ => { + return Err(Status::invalid_argument( + "first RelayFrame must be init with non-empty channel_id", + )); + } + }; + + // Claim the pending relay. Consumes the entry — it cannot be reused. + let supervisor_side = registry.claim_relay(&channel_id)?; + info!(channel_id = %channel_id, "relay stream: claimed pending relay, bridging"); + + let (mut read_half, mut write_half) = tokio::io::split(supervisor_side); + + // Supervisor → gateway: drain `inbound` and write to the DuplexStream. + let channel_id_in = channel_id.clone(); + tokio::spawn(async move { + loop { + match inbound.message().await { + Ok(Some(frame)) => { + let Some(openshell_core::proto::relay_frame::Payload::Data(data)) = + frame.payload + else { + warn!(channel_id = %channel_id_in, "relay stream: received non-data frame after init"); + break; + }; + if data.is_empty() { + continue; + } + if let Err(e) = + tokio::io::AsyncWriteExt::write_all(&mut write_half, &data).await + { + warn!(channel_id = %channel_id_in, error = %e, "relay stream: write to duplex failed"); + break; + } + } + Ok(None) => break, + Err(e) => { + warn!(channel_id = %channel_id_in, error = %e, "relay stream: inbound errored"); + break; + } + } + } + // Best-effort half-close on the write side so the reader sees EOF. + let _ = tokio::io::AsyncWriteExt::shutdown(&mut write_half).await; + }); + + // Gateway → supervisor: read the DuplexStream and emit RelayFrame::data messages. + let (out_tx, out_rx) = mpsc::channel::>(16); + let channel_id_out = channel_id.clone(); + tokio::spawn(async move { + let mut buf = vec![0u8; RELAY_STREAM_CHUNK_SIZE]; + loop { + match tokio::io::AsyncReadExt::read(&mut read_half, &mut buf).await { + Ok(0) => break, + Ok(n) => { + let chunk = RelayFrame { + payload: Some(openshell_core::proto::relay_frame::Payload::Data( + buf[..n].to_vec(), + )), + }; + if out_tx.send(Ok(chunk)).await.is_err() { + break; + } + } + Err(e) => { + warn!(channel_id = %channel_id_out, error = %e, "relay stream: read from duplex failed"); + break; + } + } + } + }); + + let stream = ReceiverStream::new(out_rx); + let stream: Pin< + Box> + Send + 'static>, + > = Box::pin(stream); + Ok(Response::new(stream)) +} + +// --------------------------------------------------------------------------- +// ConnectSupervisor gRPC handler +// --------------------------------------------------------------------------- + +pub async fn handle_connect_supervisor( + state: &Arc, + request: Request>, +) -> Result< + Response< + Pin> + Send + 'static>>, + >, + Status, +> { + let mut inbound = request.into_inner(); + + // Step 1: Wait for SupervisorHello. + let hello = match inbound.message().await? { + Some(msg) => match msg.payload { + Some(supervisor_message::Payload::Hello(hello)) => hello, + _ => return Err(Status::invalid_argument("expected SupervisorHello")), + }, + None => return Err(Status::invalid_argument("stream closed before hello")), + }; + + let sandbox_id = hello.sandbox_id.clone(); + if sandbox_id.is_empty() { + return Err(Status::invalid_argument("sandbox_id is required")); + } + require_persisted_sandbox(&state.store, &sandbox_id).await?; + + let session_id = Uuid::new_v4().to_string(); + info!( + sandbox_id = %sandbox_id, + session_id = %session_id, + instance_id = %hello.instance_id, + "supervisor session: accepted" + ); + + // Step 2: Create the outbound channel and register the session. + let (tx, rx) = mpsc::channel::(64); + let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>(); + let superseded = state.supervisor_sessions.register( + sandbox_id.clone(), + session_id.clone(), + tx.clone(), + shutdown_tx, + ); + if superseded { + info!( + sandbox_id = %sandbox_id, + session_id = %session_id, + "supervisor session: superseded previous session" + ); + } + + // Step 3: Send SessionAccepted. + let accepted = GatewayMessage { + payload: Some(gateway_message::Payload::SessionAccepted(SessionAccepted { + session_id: session_id.clone(), + heartbeat_interval_secs: HEARTBEAT_INTERVAL_SECS, + })), + }; + if tx.send(accepted).await.is_err() { + // Only evict ourselves — a faster reconnect may already have + // superseded this registration. + state + .supervisor_sessions + .remove_if_current(&sandbox_id, &session_id); + return Err(Status::internal("failed to send session accepted")); + } + + if superseded { + state + .supervisor_sessions + .replay_pending_relays(&sandbox_id, &tx) + .await; + } + + // Step 4: Spawn the session loop that reads inbound messages. + let state_clone = Arc::clone(state); + let sandbox_id_clone = sandbox_id.clone(); + tokio::spawn(async move { + run_session_loop( + &state_clone, + &sandbox_id_clone, + &session_id, + &tx, + &mut inbound, + shutdown_rx, + ) + .await; + let still_ours = state_clone + .supervisor_sessions + .remove_if_current(&sandbox_id_clone, &session_id); + if still_ours { + info!(sandbox_id = %sandbox_id_clone, session_id = %session_id, "supervisor session: ended"); + } else { + info!(sandbox_id = %sandbox_id_clone, session_id = %session_id, "supervisor session: ended (already superseded)"); + } + }); + + // Return the outbound stream. + let stream = ReceiverStream::new(rx); + let stream: Pin< + Box> + Send + 'static>, + > = Box::pin(tokio_stream::StreamExt::map(stream, Ok)); + + Ok(Response::new(stream)) +} + +async fn run_session_loop( + _state: &Arc, + sandbox_id: &str, + session_id: &str, + tx: &mpsc::Sender, + inbound: &mut tonic::Streaming, + mut shutdown_rx: oneshot::Receiver<()>, +) { + let heartbeat_interval = Duration::from_secs(u64::from(HEARTBEAT_INTERVAL_SECS)); + let mut heartbeat_timer = tokio::time::interval(heartbeat_interval); + // Skip the first immediate tick. + heartbeat_timer.tick().await; + + loop { + tokio::select! { + _ = &mut shutdown_rx => { + info!(sandbox_id = %sandbox_id, session_id = %session_id, "supervisor session: superseded by reconnect, shutting down"); + break; + } + msg = inbound.message() => { + match msg { + Ok(Some(msg)) => { + handle_supervisor_message(sandbox_id, session_id, msg); + } + Ok(None) => { + info!(sandbox_id = %sandbox_id, session_id = %session_id, "supervisor session: stream closed by supervisor"); + break; + } + Err(e) => { + warn!(sandbox_id = %sandbox_id, session_id = %session_id, error = %e, "supervisor session: stream error"); + break; + } + } + } + _ = heartbeat_timer.tick() => { + let hb = GatewayMessage { + payload: Some(gateway_message::Payload::Heartbeat( + openshell_core::proto::GatewayHeartbeat {}, + )), + }; + if tx.send(hb).await.is_err() { + info!(sandbox_id = %sandbox_id, session_id = %session_id, "supervisor session: outbound channel closed"); + break; + } + } + } + } +} + +fn handle_supervisor_message(sandbox_id: &str, session_id: &str, msg: SupervisorMessage) { + match msg.payload { + Some(supervisor_message::Payload::Heartbeat(_)) => { + // Heartbeat received — nothing to do for now. + } + Some(supervisor_message::Payload::RelayOpenResult(result)) => { + if result.success { + info!( + sandbox_id = %sandbox_id, + session_id = %session_id, + channel_id = %result.channel_id, + "supervisor session: relay opened successfully" + ); + } else { + warn!( + sandbox_id = %sandbox_id, + session_id = %session_id, + channel_id = %result.channel_id, + error = %result.error, + "supervisor session: relay open failed" + ); + } + } + Some(supervisor_message::Payload::RelayClose(close)) => { + info!( + sandbox_id = %sandbox_id, + session_id = %session_id, + channel_id = %close.channel_id, + reason = %close.reason, + "supervisor session: relay closed by supervisor" + ); + } + _ => { + warn!( + sandbox_id = %sandbox_id, + session_id = %session_id, + "supervisor session: unexpected message type" + ); + } + } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use crate::persistence::Store; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + /// Returns a shutdown sender with its receiver immediately dropped. Tests + /// that don't observe the shutdown signal can use this to satisfy the + /// `register` signature without the receiver noise. + fn make_shutdown() -> oneshot::Sender<()> { + oneshot::channel::<()>().0 + } + + fn sandbox_record(id: &str, name: &str) -> Sandbox { + Sandbox { + id: id.to_string(), + name: name.to_string(), + namespace: "default".to_string(), + ..Default::default() + } + } + + // ---- registry: register / remove ---- + + #[test] + fn registry_register_and_lookup() { + let registry = SupervisorSessionRegistry::new(); + let (tx, _rx) = mpsc::channel(1); + + assert!(!registry.register( + "sandbox-1".to_string(), + "s1".to_string(), + tx, + make_shutdown(), + )); + + let sessions = registry.sessions.lock().unwrap(); + assert!(sessions.contains_key("sandbox-1")); + } + + #[test] + fn registry_supersedes_previous_session() { + let registry = SupervisorSessionRegistry::new(); + let (tx1, _rx1) = mpsc::channel(1); + let (tx2, _rx2) = mpsc::channel(1); + + assert!(!registry.register( + "sandbox-1".to_string(), + "s1".to_string(), + tx1, + make_shutdown(), + )); + assert!(registry.register( + "sandbox-1".to_string(), + "s2".to_string(), + tx2, + make_shutdown(), + )); + } + + #[test] + fn registry_remove() { + let registry = SupervisorSessionRegistry::new(); + let (tx, _rx) = mpsc::channel(1); + registry.register( + "sandbox-1".to_string(), + "s1".to_string(), + tx, + make_shutdown(), + ); + + registry.remove("sandbox-1"); + let sessions = registry.sessions.lock().unwrap(); + assert!(!sessions.contains_key("sandbox-1")); + } + + #[test] + fn remove_if_current_removes_matching_session() { + let registry = SupervisorSessionRegistry::new(); + let (tx, _rx) = mpsc::channel(1); + registry.register("sbx".to_string(), "s1".to_string(), tx, make_shutdown()); + + assert!(registry.remove_if_current("sbx", "s1")); + assert!(!registry.sessions.lock().unwrap().contains_key("sbx")); + } + + #[test] + fn remove_if_current_ignores_stale_session_id() { + let registry = SupervisorSessionRegistry::new(); + let (tx_old, _rx_old) = mpsc::channel(1); + let (tx_new, _rx_new) = mpsc::channel(1); + + // Old session registers, then is superseded by a new session. + registry.register( + "sbx".to_string(), + "s-old".to_string(), + tx_old, + make_shutdown(), + ); + registry.register( + "sbx".to_string(), + "s-new".to_string(), + tx_new, + make_shutdown(), + ); + + // Cleanup from the old session task runs late. It must NOT evict the + // newly registered session. + assert!(!registry.remove_if_current("sbx", "s-old")); + let sessions = registry.sessions.lock().unwrap(); + assert!( + sessions.contains_key("sbx"), + "new session must still be registered" + ); + assert_eq!(sessions.get("sbx").unwrap().session_id, "s-new"); + } + + #[test] + fn remove_if_current_unknown_sandbox_is_noop() { + let registry = SupervisorSessionRegistry::new(); + assert!(!registry.remove_if_current("sbx-does-not-exist", "s1")); + } + + // ---- open_relay: happy path and wait semantics ---- + + #[tokio::test] + async fn open_relay_sends_relay_open_to_registered_session() { + let registry = SupervisorSessionRegistry::new(); + let (tx, mut rx) = mpsc::channel(4); + registry.register("sbx".to_string(), "s1".to_string(), tx, make_shutdown()); + + let (channel_id, _relay_rx) = registry + .open_relay("sbx", Duration::from_secs(1)) + .await + .expect("open_relay should succeed when session is live"); + + let msg = rx.recv().await.expect("relay open should be delivered"); + match msg.payload { + Some(gateway_message::Payload::RelayOpen(open)) => { + assert_eq!(open.channel_id, channel_id); + } + other => panic!("expected RelayOpen, got {other:?}"), + } + } + + #[tokio::test] + async fn open_relay_times_out_without_session() { + let registry = SupervisorSessionRegistry::new(); + let err = registry + .open_relay("missing", Duration::from_millis(50)) + .await + .expect_err("open_relay should time out"); + assert_eq!(err.code(), tonic::Code::Unavailable); + } + + #[tokio::test] + async fn open_relay_waits_for_session_to_appear() { + let registry = Arc::new(SupervisorSessionRegistry::new()); + let registry_for_register = Arc::clone(®istry); + + // Register the session after a small delay, shorter than the wait. + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(200)).await; + let (tx, mut rx) = mpsc::channel::(4); + // Keep the receiver alive so the send in open_relay succeeds. + tokio::spawn(async move { while rx.recv().await.is_some() {} }); + registry_for_register.register( + "sbx".to_string(), + "s1".to_string(), + tx, + make_shutdown(), + ); + }); + + let result = registry.open_relay("sbx", Duration::from_secs(2)).await; + assert!( + result.is_ok(), + "open_relay should succeed when session arrives mid-wait: {result:?}" + ); + } + + #[tokio::test] + async fn open_relay_fails_when_session_receiver_dropped() { + let registry = SupervisorSessionRegistry::new(); + let (tx, rx) = mpsc::channel::(4); + registry.register("sbx".to_string(), "s1".to_string(), tx, make_shutdown()); + + // Simulate the supervisor's stream going away between lookup and send: + // the receiver held by `ReceiverStream` is dropped. + drop(rx); + + let err = registry + .open_relay("sbx", Duration::from_secs(1)) + .await + .expect_err("open_relay should fail when mpsc is closed"); + assert_eq!(err.code(), tonic::Code::Unavailable); + // The pending-relay entry must have been cleaned up on failure. + assert!(registry.pending_relays.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn open_relay_rejects_when_global_cap_reached() { + let registry = SupervisorSessionRegistry::new(); + let (tx, _rx) = mpsc::channel::(8); + registry.register( + "sbx-a".to_string(), + "s-a".to_string(), + tx.clone(), + make_shutdown(), + ); + registry.register("sbx-b".to_string(), "s-b".to_string(), tx, make_shutdown()); + + // Pre-seed pending_relays to exactly the global cap, split across two + // sandboxes so neither hits the per-sandbox cap first. + { + let mut pending = registry.pending_relays.lock().unwrap(); + for i in 0..MAX_PENDING_RELAYS { + let (oneshot_tx, _) = oneshot::channel(); + let sandbox_id = if i % 2 == 0 { "sbx-a" } else { "sbx-b" }; + pending.insert( + format!("channel-{i}"), + PendingRelay { + sender: oneshot_tx, + sandbox_id: sandbox_id.to_string(), + created_at: Instant::now(), + }, + ); + } + } + + let err = registry + .open_relay("sbx-a", Duration::from_millis(50)) + .await + .expect_err("open_relay should reject once global cap is reached"); + assert_eq!(err.code(), tonic::Code::ResourceExhausted); + assert!(err.message().contains("gateway relay capacity")); + } + + #[tokio::test] + async fn open_relay_rejects_when_per_sandbox_cap_reached() { + let registry = SupervisorSessionRegistry::new(); + let (tx, _rx) = mpsc::channel::(8); + registry.register("sbx".to_string(), "s".to_string(), tx, make_shutdown()); + + { + let mut pending = registry.pending_relays.lock().unwrap(); + for i in 0..MAX_PENDING_RELAYS_PER_SANDBOX { + let (oneshot_tx, _) = oneshot::channel(); + pending.insert( + format!("channel-{i}"), + PendingRelay { + sender: oneshot_tx, + sandbox_id: "sbx".to_string(), + created_at: Instant::now(), + }, + ); + } + } + + let err = registry + .open_relay("sbx", Duration::from_millis(50)) + .await + .expect_err("open_relay should reject when per-sandbox cap is reached"); + assert_eq!(err.code(), tonic::Code::ResourceExhausted); + assert!(err.message().contains("per-sandbox relay limit")); + + // A different sandbox still has headroom. + let (tx2, _rx2) = mpsc::channel::(8); + registry.register( + "sbx-other".to_string(), + "s-other".to_string(), + tx2, + make_shutdown(), + ); + registry + .open_relay("sbx-other", Duration::from_millis(50)) + .await + .expect("different sandbox should still accept new relays"); + } + + #[tokio::test] + async fn open_relay_uses_newest_session_after_supersede() { + let registry = SupervisorSessionRegistry::new(); + let (tx_old, mut rx_old) = mpsc::channel::(4); + let (tx_new, mut rx_new) = mpsc::channel(4); + + // Hold a clone of the old sender so supersede doesn't close the old + // channel — that way try_recv distinguishes "no message sent" from + // "channel closed". + let _tx_old_alive = tx_old.clone(); + + registry.register( + "sbx".to_string(), + "s-old".to_string(), + tx_old, + make_shutdown(), + ); + registry.register( + "sbx".to_string(), + "s-new".to_string(), + tx_new, + make_shutdown(), + ); + + let (_channel_id, _relay_rx) = registry + .open_relay("sbx", Duration::from_secs(1)) + .await + .expect("open_relay should succeed"); + + let msg = rx_new + .recv() + .await + .expect("new session should receive RelayOpen"); + assert!(matches!( + msg.payload, + Some(gateway_message::Payload::RelayOpen(_)) + )); + + // The old session must have received no messages — the channel is + // still open but empty. + use tokio::sync::mpsc::error::TryRecvError; + match rx_old.try_recv() { + Err(TryRecvError::Empty) => {} + other => panic!("expected Empty on superseded session, got {other:?}"), + } + } + + #[tokio::test] + async fn register_signals_shutdown_to_previous_session() { + let registry = SupervisorSessionRegistry::new(); + let (tx_old, _rx_old) = mpsc::channel::(1); + let (tx_new, _rx_new) = mpsc::channel::(1); + + let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>(); + registry.register("sbx".to_string(), "s-old".to_string(), tx_old, shutdown_tx); + + // Supersede with a new session — register must fire the old session's + // shutdown signal so its task can exit and drop its tx clone. + let superseded = registry.register( + "sbx".to_string(), + "s-new".to_string(), + tx_new, + make_shutdown(), + ); + assert!(superseded, "second register should report supersede"); + + // The old session's shutdown receiver must now resolve. + shutdown_rx + .await + .expect("shutdown signal should arrive at superseded session"); + } + + #[tokio::test] + async fn replay_pending_relays_reissues_open_to_superseding_session() { + let registry = SupervisorSessionRegistry::new(); + let (tx_old, mut rx_old) = mpsc::channel::(4); + let (tx_new, mut rx_new) = mpsc::channel::(4); + + registry.register( + "sbx".to_string(), + "s-old".to_string(), + tx_old, + make_shutdown(), + ); + + let (channel_id, _relay_rx) = registry + .open_relay("sbx", Duration::from_secs(1)) + .await + .expect("open_relay should succeed"); + + let original = rx_old + .recv() + .await + .expect("old session should receive initial RelayOpen"); + assert!(matches!( + original.payload, + Some(gateway_message::Payload::RelayOpen(_)) + )); + + let superseded = registry.register( + "sbx".to_string(), + "s-new".to_string(), + tx_new, + make_shutdown(), + ); + assert!(superseded); + + registry + .replay_pending_relays("sbx", ®istry.lookup_session("sbx").unwrap()) + .await; + + let replayed = rx_new + .recv() + .await + .expect("new session should receive replayed RelayOpen"); + match replayed.payload { + Some(gateway_message::Payload::RelayOpen(open)) => { + assert_eq!(open.channel_id, channel_id); + } + other => panic!("expected RelayOpen on replay, got {other:?}"), + } + } + + #[tokio::test] + async fn require_persisted_sandbox_rejects_missing_sandbox() { + let store = Arc::new( + Store::connect("sqlite::memory:?cache=shared") + .await + .unwrap(), + ); + + let err = require_persisted_sandbox(&store, "missing") + .await + .expect_err("missing sandbox should be rejected"); + + assert_eq!(err.code(), tonic::Code::NotFound); + } + + #[tokio::test] + async fn require_persisted_sandbox_accepts_existing_sandbox() { + let store = Arc::new( + Store::connect("sqlite::memory:?cache=shared") + .await + .unwrap(), + ); + store + .put_message(&sandbox_record("sbx-1", "sandbox-one")) + .await + .expect("sandbox should persist"); + + require_persisted_sandbox(&store, "sbx-1") + .await + .expect("persisted sandbox should be accepted"); + } + + // ---- claim_relay: expiry, drop, wiring ---- + + #[test] + fn claim_relay_unknown_channel() { + let registry = SupervisorSessionRegistry::new(); + let err = registry.claim_relay("nonexistent").expect_err("should err"); + assert_eq!(err.code(), tonic::Code::NotFound); + } + + #[test] + fn claim_relay_success() { + let registry = SupervisorSessionRegistry::new(); + let (relay_tx, _relay_rx) = oneshot::channel(); + registry.pending_relays.lock().unwrap().insert( + "ch-1".to_string(), + PendingRelay { + sender: relay_tx, + sandbox_id: "sbx-test".to_string(), + created_at: Instant::now(), + }, + ); + + let result = registry.claim_relay("ch-1"); + assert!(result.is_ok()); + assert!(!registry.pending_relays.lock().unwrap().contains_key("ch-1")); + } + + #[test] + fn claim_relay_expired_returns_deadline_exceeded() { + let registry = SupervisorSessionRegistry::new(); + let (relay_tx, _relay_rx) = oneshot::channel(); + registry.pending_relays.lock().unwrap().insert( + "ch-old".to_string(), + PendingRelay { + sender: relay_tx, + sandbox_id: "sbx-test".to_string(), + created_at: Instant::now() - Duration::from_secs(60), + }, + ); + + let err = registry + .claim_relay("ch-old") + .expect_err("expired entry must fail"); + assert_eq!(err.code(), tonic::Code::DeadlineExceeded); + // Entry must have been consumed regardless. + assert!( + !registry + .pending_relays + .lock() + .unwrap() + .contains_key("ch-old") + ); + } + + #[test] + fn claim_relay_receiver_dropped_returns_internal() { + let registry = SupervisorSessionRegistry::new(); + let (relay_tx, relay_rx) = oneshot::channel::(); + drop(relay_rx); // Gateway-side waiter has given up already. + registry.pending_relays.lock().unwrap().insert( + "ch-1".to_string(), + PendingRelay { + sender: relay_tx, + sandbox_id: "sbx-test".to_string(), + created_at: Instant::now(), + }, + ); + + let err = registry + .claim_relay("ch-1") + .expect_err("should err when receiver is gone"); + assert_eq!(err.code(), tonic::Code::Internal); + } + + #[tokio::test] + async fn claim_relay_connects_both_ends() { + let registry = SupervisorSessionRegistry::new(); + let (relay_tx, relay_rx) = oneshot::channel::(); + registry.pending_relays.lock().unwrap().insert( + "ch-io".to_string(), + PendingRelay { + sender: relay_tx, + sandbox_id: "sbx-test".to_string(), + created_at: Instant::now(), + }, + ); + + let mut supervisor_side = registry.claim_relay("ch-io").expect("claim should succeed"); + let mut gateway_side = relay_rx.await.expect("gateway side should receive stream"); + + // Supervisor side writes → gateway side reads. + supervisor_side.write_all(b"hello").await.unwrap(); + let mut buf = [0u8; 5]; + gateway_side.read_exact(&mut buf).await.unwrap(); + assert_eq!(&buf, b"hello"); + + // Gateway side writes → supervisor side reads. + gateway_side.write_all(b"world").await.unwrap(); + let mut buf = [0u8; 5]; + supervisor_side.read_exact(&mut buf).await.unwrap(); + assert_eq!(&buf, b"world"); + } + + // ---- reap_expired_relays ---- + + #[test] + fn reap_expired_relays_removes_old_entries() { + let registry = SupervisorSessionRegistry::new(); + let (relay_tx, _relay_rx) = oneshot::channel(); + registry.pending_relays.lock().unwrap().insert( + "ch-old".to_string(), + PendingRelay { + sender: relay_tx, + sandbox_id: "sbx-test".to_string(), + created_at: Instant::now() - Duration::from_secs(60), + }, + ); + + registry.reap_expired_relays(); + assert!( + !registry + .pending_relays + .lock() + .unwrap() + .contains_key("ch-old") + ); + } + + #[test] + fn reap_expired_relays_keeps_fresh_entries() { + let registry = SupervisorSessionRegistry::new(); + let (relay_tx, _relay_rx) = oneshot::channel(); + registry.pending_relays.lock().unwrap().insert( + "ch-fresh".to_string(), + PendingRelay { + sender: relay_tx, + sandbox_id: "sbx-test".to_string(), + created_at: Instant::now(), + }, + ); + + registry.reap_expired_relays(); + assert!( + registry + .pending_relays + .lock() + .unwrap() + .contains_key("ch-fresh") + ); + } +} diff --git a/crates/openshell-server/tests/auth_endpoint_integration.rs b/crates/openshell-server/tests/auth_endpoint_integration.rs index 7c6545873c..12f302b63a 100644 --- a/crates/openshell-server/tests/auth_endpoint_integration.rs +++ b/crates/openshell-server/tests/auth_endpoint_integration.rs @@ -528,6 +528,9 @@ impl openshell_core::proto::open_shell_server::OpenShell for TestOpenShell { type ExecSandboxStream = tokio_stream::wrappers::ReceiverStream< Result, >; + type ConnectSupervisorStream = tokio_stream::wrappers::ReceiverStream< + Result, + >; async fn watch_sandbox( &self, @@ -663,6 +666,24 @@ impl openshell_core::proto::open_shell_server::OpenShell for TestOpenShell { { Err(tonic::Status::unimplemented("not implemented in test")) } + + async fn connect_supervisor( + &self, + _request: tonic::Request>, + ) -> Result, tonic::Status> { + Err(tonic::Status::unimplemented("not implemented in test")) + } + + type RelayStreamStream = tokio_stream::wrappers::ReceiverStream< + Result, + >; + + async fn relay_stream( + &self, + _request: tonic::Request>, + ) -> Result, tonic::Status> { + Err(tonic::Status::unimplemented("not implemented in test")) + } } /// Test 7: Plaintext server (no TLS) accepts both gRPC and HTTP. diff --git a/crates/openshell-server/tests/edge_tunnel_auth.rs b/crates/openshell-server/tests/edge_tunnel_auth.rs index 22f08434d6..e8c7e00388 100644 --- a/crates/openshell-server/tests/edge_tunnel_auth.rs +++ b/crates/openshell-server/tests/edge_tunnel_auth.rs @@ -37,13 +37,14 @@ use hyper_util::{ use openshell_core::proto::{ CreateProviderRequest, CreateSandboxRequest, CreateSshSessionRequest, CreateSshSessionResponse, DeleteProviderRequest, DeleteProviderResponse, DeleteSandboxRequest, DeleteSandboxResponse, - ExecSandboxEvent, ExecSandboxRequest, GetGatewayConfigRequest, GetGatewayConfigResponse, - GetProviderRequest, GetSandboxConfigRequest, GetSandboxConfigResponse, - GetSandboxProviderEnvironmentRequest, GetSandboxProviderEnvironmentResponse, GetSandboxRequest, - HealthRequest, HealthResponse, ListProvidersRequest, ListProvidersResponse, - ListSandboxesRequest, ListSandboxesResponse, ProviderResponse, RevokeSshSessionRequest, - RevokeSshSessionResponse, SandboxResponse, SandboxStreamEvent, ServiceStatus, - UpdateProviderRequest, WatchSandboxRequest, + ExecSandboxEvent, ExecSandboxRequest, GatewayMessage, GetGatewayConfigRequest, + GetGatewayConfigResponse, GetProviderRequest, GetSandboxConfigRequest, + GetSandboxConfigResponse, GetSandboxProviderEnvironmentRequest, + GetSandboxProviderEnvironmentResponse, GetSandboxRequest, HealthRequest, HealthResponse, + ListProvidersRequest, ListProvidersResponse, ListSandboxesRequest, ListSandboxesResponse, + ProviderResponse, RevokeSshSessionRequest, RevokeSshSessionResponse, SandboxResponse, + SandboxStreamEvent, ServiceStatus, SupervisorMessage, UpdateProviderRequest, + WatchSandboxRequest, open_shell_client::OpenShellClient, open_shell_server::{OpenShell, OpenShellServer}, }; @@ -186,6 +187,7 @@ impl OpenShell for TestOpenShell { type WatchSandboxStream = ReceiverStream>; type ExecSandboxStream = ReceiverStream>; + type ConnectSupervisorStream = ReceiverStream>; async fn watch_sandbox( &self, @@ -307,6 +309,24 @@ impl OpenShell for TestOpenShell { ) -> Result, Status> { Err(Status::unimplemented("not implemented in test")) } + + async fn connect_supervisor( + &self, + _request: tonic::Request>, + ) -> Result, Status> { + Err(Status::unimplemented("not implemented in test")) + } + + type RelayStreamStream = tokio_stream::wrappers::ReceiverStream< + Result, + >; + + async fn relay_stream( + &self, + _request: tonic::Request>, + ) -> Result, tonic::Status> { + Err(tonic::Status::unimplemented("not implemented in test")) + } } // --------------------------------------------------------------------------- diff --git a/crates/openshell-server/tests/multiplex_integration.rs b/crates/openshell-server/tests/multiplex_integration.rs index 1957c5b87d..561ea2ba72 100644 --- a/crates/openshell-server/tests/multiplex_integration.rs +++ b/crates/openshell-server/tests/multiplex_integration.rs @@ -11,13 +11,14 @@ use hyper_util::{ use openshell_core::proto::{ CreateProviderRequest, CreateSandboxRequest, CreateSshSessionRequest, CreateSshSessionResponse, DeleteProviderRequest, DeleteProviderResponse, DeleteSandboxRequest, DeleteSandboxResponse, - ExecSandboxEvent, ExecSandboxRequest, GetGatewayConfigRequest, GetGatewayConfigResponse, - GetProviderRequest, GetSandboxConfigRequest, GetSandboxConfigResponse, - GetSandboxProviderEnvironmentRequest, GetSandboxProviderEnvironmentResponse, GetSandboxRequest, - HealthRequest, HealthResponse, ListProvidersRequest, ListProvidersResponse, - ListSandboxesRequest, ListSandboxesResponse, ProviderResponse, RevokeSshSessionRequest, - RevokeSshSessionResponse, SandboxResponse, SandboxStreamEvent, ServiceStatus, - UpdateProviderRequest, WatchSandboxRequest, + ExecSandboxEvent, ExecSandboxRequest, GatewayMessage, GetGatewayConfigRequest, + GetGatewayConfigResponse, GetProviderRequest, GetSandboxConfigRequest, + GetSandboxConfigResponse, GetSandboxProviderEnvironmentRequest, + GetSandboxProviderEnvironmentResponse, GetSandboxRequest, HealthRequest, HealthResponse, + ListProvidersRequest, ListProvidersResponse, ListSandboxesRequest, ListSandboxesResponse, + ProviderResponse, RevokeSshSessionRequest, RevokeSshSessionResponse, SandboxResponse, + SandboxStreamEvent, ServiceStatus, SupervisorMessage, UpdateProviderRequest, + WatchSandboxRequest, open_shell_client::OpenShellClient, open_shell_server::{OpenShell, OpenShellServer}, }; @@ -154,6 +155,7 @@ impl OpenShell for TestOpenShell { type WatchSandboxStream = ReceiverStream>; type ExecSandboxStream = ReceiverStream>; + type ConnectSupervisorStream = ReceiverStream>; async fn watch_sandbox( &self, @@ -275,6 +277,24 @@ impl OpenShell for TestOpenShell { ) -> Result, Status> { Err(Status::unimplemented("not implemented in test")) } + + async fn connect_supervisor( + &self, + _request: tonic::Request>, + ) -> Result, Status> { + Err(Status::unimplemented("not implemented in test")) + } + + type RelayStreamStream = tokio_stream::wrappers::ReceiverStream< + Result, + >; + + async fn relay_stream( + &self, + _request: tonic::Request>, + ) -> Result, tonic::Status> { + Err(tonic::Status::unimplemented("not implemented in test")) + } } #[tokio::test] diff --git a/crates/openshell-server/tests/multiplex_tls_integration.rs b/crates/openshell-server/tests/multiplex_tls_integration.rs index 98d5d62563..dc51e61182 100644 --- a/crates/openshell-server/tests/multiplex_tls_integration.rs +++ b/crates/openshell-server/tests/multiplex_tls_integration.rs @@ -13,13 +13,14 @@ use hyper_util::{ use openshell_core::proto::{ CreateProviderRequest, CreateSandboxRequest, CreateSshSessionRequest, CreateSshSessionResponse, DeleteProviderRequest, DeleteProviderResponse, DeleteSandboxRequest, DeleteSandboxResponse, - ExecSandboxEvent, ExecSandboxRequest, GetGatewayConfigRequest, GetGatewayConfigResponse, - GetProviderRequest, GetSandboxConfigRequest, GetSandboxConfigResponse, - GetSandboxProviderEnvironmentRequest, GetSandboxProviderEnvironmentResponse, GetSandboxRequest, - HealthRequest, HealthResponse, ListProvidersRequest, ListProvidersResponse, - ListSandboxesRequest, ListSandboxesResponse, ProviderResponse, RevokeSshSessionRequest, - RevokeSshSessionResponse, SandboxResponse, SandboxStreamEvent, ServiceStatus, - UpdateProviderRequest, WatchSandboxRequest, + ExecSandboxEvent, ExecSandboxRequest, GatewayMessage, GetGatewayConfigRequest, + GetGatewayConfigResponse, GetProviderRequest, GetSandboxConfigRequest, + GetSandboxConfigResponse, GetSandboxProviderEnvironmentRequest, + GetSandboxProviderEnvironmentResponse, GetSandboxRequest, HealthRequest, HealthResponse, + ListProvidersRequest, ListProvidersResponse, ListSandboxesRequest, ListSandboxesResponse, + ProviderResponse, RevokeSshSessionRequest, RevokeSshSessionResponse, SandboxResponse, + SandboxStreamEvent, ServiceStatus, SupervisorMessage, UpdateProviderRequest, + WatchSandboxRequest, open_shell_client::OpenShellClient, open_shell_server::{OpenShell, OpenShellServer}, }; @@ -167,6 +168,7 @@ impl OpenShell for TestOpenShell { type WatchSandboxStream = ReceiverStream>; type ExecSandboxStream = ReceiverStream>; + type ConnectSupervisorStream = ReceiverStream>; async fn watch_sandbox( &self, @@ -288,6 +290,24 @@ impl OpenShell for TestOpenShell { ) -> Result, Status> { Err(Status::unimplemented("not implemented in test")) } + + async fn connect_supervisor( + &self, + _request: tonic::Request>, + ) -> Result, Status> { + Err(Status::unimplemented("not implemented in test")) + } + + type RelayStreamStream = tokio_stream::wrappers::ReceiverStream< + Result, + >; + + async fn relay_stream( + &self, + _request: tonic::Request>, + ) -> Result, tonic::Status> { + Err(tonic::Status::unimplemented("not implemented in test")) + } } /// PKI bundle: CA cert, server cert+key, client cert+key. diff --git a/crates/openshell-server/tests/supervisor_relay_integration.rs b/crates/openshell-server/tests/supervisor_relay_integration.rs new file mode 100644 index 0000000000..85d263223b --- /dev/null +++ b/crates/openshell-server/tests/supervisor_relay_integration.rs @@ -0,0 +1,572 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Integration tests for the supervisor relay gRPC path. +//! +//! Stands up an in-process tonic server hosting the real `handle_relay_stream` +//! handler, plus a mock "supervisor" client that calls `relay_stream` over a +//! real `Channel`. Exercises the wire contract (typed `RelayFrame { Init | Data }`), +//! `SupervisorSessionRegistry::open_relay` → `claim_relay` pairing, and the +//! bidirectional byte bridge inside the handler. +//! +//! These tests complement the unit tests in `supervisor_session.rs` (which +//! exercise registry state only) and the live cluster tests (which exercise +//! the full CLI → gateway → sandbox path). They catch regressions in the gRPC +//! wire layer that unit tests can't see and that are expensive to catch in +//! E2E. + +use std::sync::Arc; +use std::time::Duration; + +use hyper_util::{ + rt::{TokioExecutor, TokioIo}, + server::conn::auto::Builder, +}; +use openshell_core::proto::{ + GatewayMessage, RelayFrame, RelayInit, SupervisorMessage, + open_shell_client::OpenShellClient, + open_shell_server::{OpenShell, OpenShellServer}, +}; +use openshell_server::supervisor_session::SupervisorSessionRegistry; +use openshell_server::{MultiplexedService, health_router}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpListener; +use tokio::sync::mpsc; +use tokio_stream::StreamExt; +use tokio_stream::wrappers::ReceiverStream; +use tonic::transport::{Channel, Endpoint}; +use tonic::{Response, Status}; + +// --------------------------------------------------------------------------- +// Gateway service: only relay_stream does real work; everything else stubs. +// --------------------------------------------------------------------------- + +#[derive(Clone)] +struct RelayGateway { + registry: Arc, +} + +#[tonic::async_trait] +impl OpenShell for RelayGateway { + type RelayStreamStream = std::pin::Pin< + Box> + Send + 'static>, + >; + + async fn relay_stream( + &self, + request: tonic::Request>, + ) -> Result, Status> { + openshell_server::supervisor_session::handle_relay_stream(&self.registry, request).await + } + + // ------ unused stubs ------ + + type ConnectSupervisorStream = ReceiverStream>; + async fn connect_supervisor( + &self, + _: tonic::Request>, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + + type WatchSandboxStream = + ReceiverStream>; + async fn watch_sandbox( + &self, + _: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + + type ExecSandboxStream = + ReceiverStream>; + async fn exec_sandbox( + &self, + _: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + + async fn health( + &self, + _: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + async fn create_sandbox( + &self, + _: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + async fn get_sandbox( + &self, + _: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + async fn list_sandboxes( + &self, + _: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + async fn delete_sandbox( + &self, + _: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + async fn get_sandbox_config( + &self, + _: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + async fn get_gateway_config( + &self, + _: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + async fn get_sandbox_provider_environment( + &self, + _: tonic::Request, + ) -> Result, Status> + { + Err(Status::unimplemented("unused")) + } + async fn create_ssh_session( + &self, + _: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + async fn revoke_ssh_session( + &self, + _: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + async fn create_provider( + &self, + _: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + async fn update_provider( + &self, + _: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + async fn get_provider( + &self, + _: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + async fn list_providers( + &self, + _: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + async fn delete_provider( + &self, + _: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + async fn update_config( + &self, + _: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + async fn get_sandbox_policy_status( + &self, + _: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + async fn list_sandbox_policies( + &self, + _: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + async fn report_policy_status( + &self, + _: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + async fn get_sandbox_logs( + &self, + _: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + async fn push_sandbox_logs( + &self, + _: tonic::Request>, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + async fn submit_policy_analysis( + &self, + _: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + async fn get_draft_policy( + &self, + _: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + async fn approve_draft_chunk( + &self, + _: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + async fn reject_draft_chunk( + &self, + _: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + async fn approve_all_draft_chunks( + &self, + _: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + async fn edit_draft_chunk( + &self, + _: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + async fn undo_draft_chunk( + &self, + _: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + async fn clear_draft_chunks( + &self, + _: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } + async fn get_draft_history( + &self, + _: tonic::Request, + ) -> Result, Status> { + Err(Status::unimplemented("unused")) + } +} + +// --------------------------------------------------------------------------- +// Test harness +// --------------------------------------------------------------------------- + +async fn spawn_gateway(registry: Arc) -> Channel { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + + let grpc = OpenShellServer::new(RelayGateway { registry }); + let service = MultiplexedService::new(grpc, health_router()); + + tokio::spawn(async move { + loop { + let Ok((stream, _)) = listener.accept().await else { + continue; + }; + let svc = service.clone(); + tokio::spawn(async move { + let _ = Builder::new(TokioExecutor::new()) + .serve_connection(TokioIo::new(stream), svc) + .await; + }); + } + }); + + Endpoint::from_shared(format!("http://{addr}")) + .unwrap() + .connect() + .await + .expect("client connect") +} + +fn register_session( + registry: &SupervisorSessionRegistry, + sandbox_id: &str, +) -> mpsc::Receiver { + register_session_with_capacity(registry, sandbox_id, 8) +} + +fn register_session_with_capacity( + registry: &SupervisorSessionRegistry, + sandbox_id: &str, + capacity: usize, +) -> mpsc::Receiver { + let (tx, rx) = mpsc::channel(capacity); + let (shutdown_tx, _shutdown_rx) = tokio::sync::oneshot::channel::<()>(); + registry.register( + sandbox_id.to_string(), + "sess-1".to_string(), + tx, + shutdown_tx, + ); + rx +} + +/// Mock supervisor that opens a `RelayStream`, sends `Init`, then echoes every +/// data frame it receives. Returns when the gateway drops the stream or when +/// the supervisor's own outbound channel closes. +async fn run_echo_supervisor(channel: Channel, channel_id: String) { + let mut client = OpenShellClient::new(channel); + let (out_tx, out_rx) = mpsc::channel::(16); + let outbound = ReceiverStream::new(out_rx); + + out_tx + .send(RelayFrame { + payload: Some(openshell_core::proto::relay_frame::Payload::Init( + RelayInit { channel_id }, + )), + }) + .await + .expect("send init"); + + let response = client + .relay_stream(outbound) + .await + .expect("relay_stream rpc"); + let mut inbound = response.into_inner(); + + while let Some(msg) = inbound.next().await { + let Ok(frame) = msg else { break }; + let Some(openshell_core::proto::relay_frame::Payload::Data(data)) = frame.payload else { + continue; + }; + let echoed = RelayFrame { + payload: Some(openshell_core::proto::relay_frame::Payload::Data(data)), + }; + if out_tx.send(echoed).await.is_err() { + break; + } + } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[tokio::test] +async fn relay_round_trips_bytes() { + let registry = Arc::new(SupervisorSessionRegistry::new()); + let channel = spawn_gateway(Arc::clone(®istry)).await; + let mut session_rx = register_session(®istry, "sbx"); + + let (channel_id, relay_rx) = registry + .open_relay("sbx", Duration::from_secs(2)) + .await + .expect("open_relay"); + + let opened = match session_rx.recv().await.expect("RelayOpen").payload { + Some(openshell_core::proto::gateway_message::Payload::RelayOpen(r)) => r.channel_id, + other => panic!("expected RelayOpen, got {other:?}"), + }; + assert_eq!(opened, channel_id); + + tokio::spawn(run_echo_supervisor(channel, channel_id)); + + let relay = relay_rx.await.expect("relay duplex"); + let (mut read_half, mut write_half) = tokio::io::split(relay); + + write_half.write_all(b"hello relay").await.expect("write"); + write_half.flush().await.expect("flush"); + + let mut buf = [0u8; 11]; + read_half.read_exact(&mut buf).await.expect("read echoed"); + assert_eq!(&buf, b"hello relay"); +} + +#[tokio::test] +async fn relay_closes_cleanly_when_gateway_drops() { + let registry = Arc::new(SupervisorSessionRegistry::new()); + let channel = spawn_gateway(Arc::clone(®istry)).await; + let mut session_rx = register_session(®istry, "sbx"); + + let (channel_id, relay_rx) = registry + .open_relay("sbx", Duration::from_secs(2)) + .await + .expect("open_relay"); + let _ = session_rx.recv().await.expect("RelayOpen"); + + let supervisor = tokio::spawn(run_echo_supervisor(channel, channel_id)); + + let relay = relay_rx.await.expect("relay duplex"); + drop(relay); + + // The supervisor's inbound stream should terminate shortly after the + // gateway side drops — its echo loop exits and the task finishes. + tokio::time::timeout(Duration::from_secs(5), supervisor) + .await + .expect("supervisor should terminate after gateway drop") + .expect("supervisor task"); +} + +#[tokio::test] +async fn relay_sees_eof_when_supervisor_closes() { + let registry = Arc::new(SupervisorSessionRegistry::new()); + let channel = spawn_gateway(Arc::clone(®istry)).await; + let mut session_rx = register_session(®istry, "sbx"); + + let (channel_id, relay_rx) = registry + .open_relay("sbx", Duration::from_secs(2)) + .await + .expect("open_relay"); + let _ = session_rx.recv().await.expect("RelayOpen"); + + // Supervisor sends init, then drops its outbound sender → gateway reader + // should see EOF. + let supervisor = { + let channel_id = channel_id.clone(); + tokio::spawn(async move { + let mut client = OpenShellClient::new(channel); + let (out_tx, out_rx) = mpsc::channel::(4); + let outbound = ReceiverStream::new(out_rx); + out_tx + .send(RelayFrame { + payload: Some(openshell_core::proto::relay_frame::Payload::Init( + RelayInit { channel_id }, + )), + }) + .await + .expect("send init"); + let _response = client.relay_stream(outbound).await.expect("rpc"); + drop(out_tx); + tokio::time::sleep(Duration::from_millis(200)).await; + }) + }; + + let relay = relay_rx.await.expect("relay duplex"); + let (mut read_half, _write_half) = tokio::io::split(relay); + let mut buf = [0u8; 16]; + let n = tokio::time::timeout(Duration::from_secs(5), read_half.read(&mut buf)) + .await + .expect("read should complete") + .expect("read ok"); + assert_eq!(n, 0, "gateway-side read should see EOF"); + + supervisor.await.expect("supervisor task"); +} + +#[tokio::test] +async fn open_relay_times_out_when_no_session() { + let registry = Arc::new(SupervisorSessionRegistry::new()); + let _channel = spawn_gateway(Arc::clone(®istry)).await; + + let err = registry + .open_relay("missing", Duration::from_millis(100)) + .await + .expect_err("should time out"); + assert_eq!(err.code(), tonic::Code::Unavailable); +} + +#[tokio::test] +async fn concurrent_relays_multiplex_independently() { + let registry = Arc::new(SupervisorSessionRegistry::new()); + let channel = spawn_gateway(Arc::clone(®istry)).await; + let mut session_rx = register_session(®istry, "sbx"); + + let (id_a, rx_a) = registry + .open_relay("sbx", Duration::from_secs(2)) + .await + .expect("open_relay a"); + let _ = session_rx.recv().await.expect("RelayOpen a"); + + let (id_b, rx_b) = registry + .open_relay("sbx", Duration::from_secs(2)) + .await + .expect("open_relay b"); + let _ = session_rx.recv().await.expect("RelayOpen b"); + assert_ne!(id_a, id_b); + + tokio::spawn(run_echo_supervisor(channel.clone(), id_a)); + tokio::spawn(run_echo_supervisor(channel, id_b)); + + let relay_a = rx_a.await.expect("relay a"); + let relay_b = rx_b.await.expect("relay b"); + + let (mut ra, mut wa) = tokio::io::split(relay_a); + let (mut rb, mut wb) = tokio::io::split(relay_b); + + wa.write_all(b"stream-A").await.unwrap(); + wb.write_all(b"stream-B").await.unwrap(); + wa.flush().await.unwrap(); + wb.flush().await.unwrap(); + + let mut buf_a = [0u8; 8]; + let mut buf_b = [0u8; 8]; + ra.read_exact(&mut buf_a).await.unwrap(); + rb.read_exact(&mut buf_b).await.unwrap(); + assert_eq!(&buf_a, b"stream-A"); + assert_eq!(&buf_b, b"stream-B"); +} + +/// Bursts more `open_relay` calls than the per-sandbox cap allows in parallel +/// and asserts the registry enforces the ceiling cleanly. A well-behaved +/// caller inside the cap still succeeds; overflow calls return `ResourceExhausted` +/// rather than racing the pending map into an inconsistent state. +#[tokio::test] +async fn open_relay_enforces_per_sandbox_cap_under_concurrent_burst() { + let registry = Arc::new(SupervisorSessionRegistry::new()); + let _channel = spawn_gateway(Arc::clone(®istry)).await; + // Oversized mpsc so the session doesn't backpressure the burst — the cap, + // not the channel, is what we're testing. + let _session_rx = register_session_with_capacity(®istry, "sbx", 256); + + // Fire 64 concurrent opens. Per-sandbox cap is 32, global cap is 256, + // so exactly 32 should succeed and 32 should be rejected with + // `ResourceExhausted` carrying the per-sandbox message. + let mut handles = Vec::with_capacity(64); + for _ in 0..64 { + let r = Arc::clone(®istry); + handles.push(tokio::spawn(async move { + r.open_relay("sbx", Duration::from_secs(1)).await + })); + } + + let mut ok = 0usize; + let mut exhausted = 0usize; + for h in handles { + match h.await.expect("task joined") { + Ok(_pair) => ok += 1, + Err(status) if status.code() == tonic::Code::ResourceExhausted => { + assert!( + status.message().contains("per-sandbox relay limit"), + "expected per-sandbox error message, got: {}", + status.message() + ); + exhausted += 1; + } + Err(other) => panic!("unexpected open_relay error: {other:?}"), + } + } + assert_eq!(ok, 32, "exactly per-sandbox cap should succeed"); + assert_eq!(exhausted, 32, "overflow should be rejected, not dropped"); + + // A different sandbox still has headroom — the per-sandbox cap doesn't + // leak onto unrelated tenants. + let _other_rx = register_session_with_capacity(®istry, "sbx-other", 8); + registry + .open_relay("sbx-other", Duration::from_secs(1)) + .await + .expect("other sandbox should not be affected by sbx cap"); +} diff --git a/crates/openshell-server/tests/ws_tunnel_integration.rs b/crates/openshell-server/tests/ws_tunnel_integration.rs index 54a7354c86..584f092818 100644 --- a/crates/openshell-server/tests/ws_tunnel_integration.rs +++ b/crates/openshell-server/tests/ws_tunnel_integration.rs @@ -40,13 +40,14 @@ use hyper_util::{ use openshell_core::proto::{ CreateProviderRequest, CreateSandboxRequest, CreateSshSessionRequest, CreateSshSessionResponse, DeleteProviderRequest, DeleteProviderResponse, DeleteSandboxRequest, DeleteSandboxResponse, - ExecSandboxEvent, ExecSandboxRequest, GetGatewayConfigRequest, GetGatewayConfigResponse, - GetProviderRequest, GetSandboxConfigRequest, GetSandboxConfigResponse, - GetSandboxProviderEnvironmentRequest, GetSandboxProviderEnvironmentResponse, GetSandboxRequest, - HealthRequest, HealthResponse, ListProvidersRequest, ListProvidersResponse, - ListSandboxesRequest, ListSandboxesResponse, ProviderResponse, RevokeSshSessionRequest, - RevokeSshSessionResponse, SandboxResponse, SandboxStreamEvent, ServiceStatus, - UpdateProviderRequest, WatchSandboxRequest, + ExecSandboxEvent, ExecSandboxRequest, GatewayMessage, GetGatewayConfigRequest, + GetGatewayConfigResponse, GetProviderRequest, GetSandboxConfigRequest, + GetSandboxConfigResponse, GetSandboxProviderEnvironmentRequest, + GetSandboxProviderEnvironmentResponse, GetSandboxRequest, HealthRequest, HealthResponse, + ListProvidersRequest, ListProvidersResponse, ListSandboxesRequest, ListSandboxesResponse, + ProviderResponse, RevokeSshSessionRequest, RevokeSshSessionResponse, SandboxResponse, + SandboxStreamEvent, ServiceStatus, SupervisorMessage, UpdateProviderRequest, + WatchSandboxRequest, open_shell_client::OpenShellClient, open_shell_server::{OpenShell, OpenShellServer}, }; @@ -180,6 +181,7 @@ impl OpenShell for TestOpenShell { type WatchSandboxStream = ReceiverStream>; type ExecSandboxStream = ReceiverStream>; + type ConnectSupervisorStream = ReceiverStream>; async fn watch_sandbox( &self, @@ -301,6 +303,24 @@ impl OpenShell for TestOpenShell { ) -> Result, Status> { Err(Status::unimplemented("not implemented in test")) } + + async fn connect_supervisor( + &self, + _request: tonic::Request>, + ) -> Result, Status> { + Err(Status::unimplemented("not implemented in test")) + } + + type RelayStreamStream = tokio_stream::wrappers::ReceiverStream< + Result, + >; + + async fn relay_stream( + &self, + _request: tonic::Request>, + ) -> Result, tonic::Status> { + Err(tonic::Status::unimplemented("not implemented in test")) + } } // --------------------------------------------------------------------------- diff --git a/docs/observability/logging.mdx b/docs/observability/logging.mdx index 2053ba4b2b..5f5aadf039 100644 --- a/docs/observability/logging.mdx +++ b/docs/observability/logging.mdx @@ -134,13 +134,13 @@ Proxy and SSH servers ready: ```text OCSF NET:LISTEN [INFO] 10.200.0.1:3128 -OCSF SSH:LISTEN [INFO] 0.0.0.0:2222 +OCSF SSH:LISTEN [INFO] ``` -An SSH handshake accepted, one event per connection: +An SSH connection accepted (one event per invocation, arriving over the supervisor's Unix socket, so there is no network peer address to log): ```text -OCSF SSH:OPEN [INFO] ALLOWED 10.42.0.52:42706 [auth:NSSH1] +OCSF SSH:OPEN [INFO] ALLOWED ``` A process launched inside the sandbox: diff --git a/docs/security/best-practices.mdx b/docs/security/best-practices.mdx index 45a521fbf8..87faae4a3f 100644 --- a/docs/security/best-practices.mdx +++ b/docs/security/best-practices.mdx @@ -254,7 +254,7 @@ OpenShell generates a cluster CA at bootstrap and distributes it through Kuberne ### SSH Tunnel Authentication -SSH connections to sandboxes pass through the gateway's HTTP CONNECT tunnel with token-based authentication and HMAC-SHA256 handshake verification (NSSH1 protocol). +SSH connections to sandboxes travel through the gateway over the sandbox's existing mTLS session. On top of the gateway's transport authentication (mTLS by default), each SSH connect call also carries a short-lived session token scoped to a specific sandbox. The sandbox never exposes an SSH port on the network — its SSH daemon listens on a local Unix socket that only the sandbox's own supervisor process can reach. | Aspect | Detail | |---|---| diff --git a/proto/compute_driver.proto b/proto/compute_driver.proto index 53b0ac27d5..68af695e51 100644 --- a/proto/compute_driver.proto +++ b/proto/compute_driver.proto @@ -38,10 +38,6 @@ service ComputeDriver { // Tear down platform resources for a sandbox. rpc DeleteSandbox(DeleteSandboxRequest) returns (DeleteSandboxResponse); - // Resolve the current endpoint for sandbox exec/SSH transport. - rpc ResolveSandboxEndpoint(ResolveSandboxEndpointRequest) - returns (ResolveSandboxEndpointResponse); - // Stream sandbox observations from the platform. rpc WatchSandboxes(WatchSandboxesRequest) returns (stream WatchSandboxesEvent); } @@ -238,27 +234,6 @@ message DeleteSandboxResponse { bool deleted = 1; } -message ResolveSandboxEndpointRequest { - // Sandbox to resolve for exec or SSH connectivity. - DriverSandbox sandbox = 1; -} - -message SandboxEndpoint { - oneof target { - // Direct IP address for the sandbox endpoint. - string ip = 1; - // DNS host name for the sandbox endpoint. - string host = 2; - } - // TCP port for the sandbox endpoint. - uint32 port = 3; -} - -message ResolveSandboxEndpointResponse { - // Current endpoint the gateway should use to reach the sandbox. - SandboxEndpoint endpoint = 1; -} - message WatchSandboxesRequest {} message WatchSandboxesSandboxEvent { diff --git a/proto/openshell.proto b/proto/openshell.proto index 895b0d1aa9..2434f1a806 100644 --- a/proto/openshell.proto +++ b/proto/openshell.proto @@ -91,6 +91,27 @@ service OpenShell { // Push sandbox supervisor logs to the server (client-streaming). rpc PushSandboxLogs(stream PushSandboxLogsRequest) returns (PushSandboxLogsResponse); + // Persistent supervisor-to-gateway session (bidirectional streaming). + // + // The supervisor opens this stream at startup and keeps it alive for the + // sandbox lifetime. The gateway uses it to coordinate relay channels for + // SSH connect and ExecSandbox. Raw SSH bytes flow over RelayStream calls + // (separate HTTP/2 streams on the same connection), not over this stream. + rpc ConnectSupervisor(stream SupervisorMessage) returns (stream GatewayMessage); + + // Raw byte relay between supervisor and gateway. + // + // The supervisor initiates this call after receiving a RelayOpen message + // on its ConnectSupervisor stream. The first RelayFrame carries a + // RelayInit with the channel_id to associate the new HTTP/2 stream with + // the pending relay slot on the gateway. Subsequent frames carry raw bytes in either + // direction between the gateway-side waiter (ssh_tunnel / exec handler) + // and the supervisor-side local SSH daemon bridge. + // + // This rides the same TCP+TLS+HTTP/2 connection as ConnectSupervisor — + // no new TLS handshake, no reverse HTTP CONNECT. + rpc RelayStream(stream RelayFrame) returns (stream RelayFrame); + // Watch a sandbox and stream updates. // // This stream can include: @@ -761,6 +782,105 @@ message GetSandboxLogsResponse { uint32 buffer_total = 2; } +// --------------------------------------------------------------------------- +// Supervisor session messages +// --------------------------------------------------------------------------- + +// Envelope for supervisor-to-gateway messages on the ConnectSupervisor stream. +message SupervisorMessage { + oneof payload { + SupervisorHello hello = 1; + SupervisorHeartbeat heartbeat = 2; + RelayOpenResult relay_open_result = 3; + RelayClose relay_close = 4; + } +} + +// Envelope for gateway-to-supervisor messages on the ConnectSupervisor stream. +message GatewayMessage { + oneof payload { + SessionAccepted session_accepted = 1; + SessionRejected session_rejected = 2; + GatewayHeartbeat heartbeat = 3; + RelayOpen relay_open = 4; + RelayClose relay_close = 5; + } +} + +// Supervisor identifies itself and the sandbox it manages. +message SupervisorHello { + // Sandbox ID this supervisor manages. + string sandbox_id = 1; + // Supervisor instance ID (e.g. boot id or process epoch). + string instance_id = 2; +} + +// Gateway accepts the supervisor session. +message SessionAccepted { + // Gateway-assigned session ID for this connection. + string session_id = 1; + // Recommended heartbeat interval in seconds. + uint32 heartbeat_interval_secs = 2; +} + +// Gateway rejects the supervisor session. +message SessionRejected { + // Human-readable rejection reason. + string reason = 1; +} + +// Supervisor heartbeat. +message SupervisorHeartbeat {} + +// Gateway heartbeat. +message GatewayHeartbeat {} + +// Gateway requests the supervisor to open a relay channel. +// +// On receiving this, the supervisor should initiate a RelayStream RPC to +// the gateway, sending a RelayInit in the first RelayFrame to associate +// the new HTTP/2 stream with the pending relay slot. The supervisor +// bridges that stream to the local SSH daemon. +message RelayOpen { + // Gateway-allocated channel identifier (UUID). + string channel_id = 1; +} + +// Initial RelayStream frame sent by the supervisor to claim a pending relay. +message RelayInit { + // Gateway-allocated channel identifier (UUID). + string channel_id = 1; +} + +// A single frame on the RelayStream RPC. +// +// The supervisor MUST send `init` as the first frame. All subsequent frames +// in either direction carry raw bytes in `data`. +message RelayFrame { + oneof payload { + RelayInit init = 1; + bytes data = 2; + } +} + +// Supervisor reports the result of a relay open request. +message RelayOpenResult { + // Channel identifier from the RelayOpen request. + string channel_id = 1; + // True if the relay was successfully established. + bool success = 2; + // Error message if success is false. + string error = 3; +} + +// Either side requests closure of a relay channel. +message RelayClose { + // Channel identifier to close. + string channel_id = 1; + // Optional reason for closure. + string reason = 2; +} + // --------------------------------------------------------------------------- // Service status // --------------------------------------------------------------------------- From ba56206f6a8e6eac5adafce69804307425456380 Mon Sep 17 00:00:00 2001 From: Seth Jennings Date: Tue, 21 Apr 2026 12:38:43 -0500 Subject: [PATCH 020/142] feat(server): add request-level logging via tower-http TraceLayer (#895) Wrap gRPC and HTTP services with TraceLayer to log method, path, status, and latency for every request. Health/readiness probe endpoints are logged at debug level to reduce noise. --- crates/openshell-server/src/multiplex.rs | 47 +++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/crates/openshell-server/src/multiplex.rs b/crates/openshell-server/src/multiplex.rs index 0f6b9449fd..841aa92630 100644 --- a/crates/openshell-server/src/multiplex.rs +++ b/crates/openshell-server/src/multiplex.rs @@ -22,8 +22,11 @@ use std::future::Future; use std::pin::Pin; use std::sync::Arc; use std::task::{Context, Poll}; +use std::time::Duration; use tokio::io::{AsyncRead, AsyncWrite}; -use tower::ServiceExt; +use tower::{ServiceBuilder, ServiceExt}; +use tower_http::trace::TraceLayer; +use tracing::Span; use crate::{OpenShellService, ServerState, http_router, inference::InferenceService}; @@ -60,6 +63,23 @@ impl MultiplexService { let grpc_service = GrpcRouter::new(openshell, inference); let http_service = http_router(self.state.clone()); + let grpc_service = ServiceBuilder::new() + .layer( + TraceLayer::new_for_http() + .make_span_with(make_request_span) + .on_request(()) + .on_response(log_response), + ) + .service(grpc_service); + let http_service = ServiceBuilder::new() + .layer( + TraceLayer::new_for_http() + .make_span_with(make_request_span) + .on_request(()) + .on_response(log_response), + ) + .service(http_service); + let service = MultiplexedService::new(grpc_service, http_service); // HTTP/2 adaptive flow control. Default windows (64 KiB / 64 KiB) @@ -213,6 +233,31 @@ where } } +fn make_request_span(req: &Request) -> Span { + let path = req.uri().path(); + if matches!(path, "/health" | "/healthz" | "/readyz") { + tracing::debug_span!( + "request", + method = %req.method(), + path, + ) + } else { + tracing::info_span!( + "request", + method = %req.method(), + path, + ) + } +} + +fn log_response(res: &Response, latency: Duration, _span: &Span) { + tracing::info!( + status = res.status().as_u16(), + latency_ms = latency.as_millis(), + "response" + ); +} + /// Boxed body type for uniform handling. pub struct BoxBody( http_body_util::combinators::UnsyncBoxBody>, From bd113957c371c226662b9e615459620a4c290468 Mon Sep 17 00:00:00 2001 From: Seth Jennings Date: Tue, 21 Apr 2026 15:26:09 -0500 Subject: [PATCH 021/142] feat(server): serve health endpoints on separate unauthenticated port (#903) Move /health, /healthz, and /readyz to a dedicated plaintext HTTP port (default 8081) so Kubernetes probes work without mTLS client certificates. - Add health_bind_address to Config with --health-port CLI arg - Spawn standalone axum::serve for health_router on the health port - Remove health routes from the main multiplexed HTTP router - Update Helm statefulset probes from tcpSocket to httpGet on health port - Fix cluster-healthcheck.sh to open/close TCP without sending data, avoiding InvalidContentType TLS errors in the gateway log --- crates/openshell-core/src/config.rs | 15 ++++++++++++++ crates/openshell-server/src/cli.rs | 13 ++++++++++++ crates/openshell-server/src/http.rs | 3 +-- crates/openshell-server/src/lib.rs | 16 +++++++++++++++ deploy/docker/cluster-healthcheck.sh | 4 +++- .../helm/openshell/templates/statefulset.yaml | 20 +++++++++++++------ deploy/helm/openshell/values.yaml | 1 + 7 files changed, 63 insertions(+), 9 deletions(-) diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index 7a9141c520..3217a783d7 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -57,6 +57,10 @@ pub struct Config { #[serde(default = "default_bind_address")] pub bind_address: SocketAddr, + /// Address to bind the unauthenticated health endpoint to. + #[serde(default = "default_health_bind_address")] + pub health_bind_address: SocketAddr, + /// Log level (trace, debug, info, warn, error). #[serde(default = "default_log_level")] pub log_level: String, @@ -176,6 +180,7 @@ impl Config { pub fn new(tls: Option) -> Self { Self { bind_address: default_bind_address(), + health_bind_address: default_health_bind_address(), log_level: default_log_level(), tls, database_url: String::new(), @@ -203,6 +208,12 @@ impl Config { self } + #[must_use] + pub const fn with_health_bind_address(mut self, addr: SocketAddr) -> Self { + self.health_bind_address = addr; + self + } + /// Create a new configuration with the given log level. #[must_use] pub fn with_log_level(mut self, level: impl Into) -> Self { @@ -316,6 +327,10 @@ fn default_bind_address() -> SocketAddr { "0.0.0.0:8080".parse().expect("valid default address") } +fn default_health_bind_address() -> SocketAddr { + "0.0.0.0:8081".parse().expect("valid default address") +} + fn default_log_level() -> String { "info".to_string() } diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index 8eff8c8578..0b64de803a 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -23,6 +23,10 @@ struct Args { #[arg(long, default_value_t = 8080, env = "OPENSHELL_SERVER_PORT")] port: u16, + /// Port for unauthenticated health endpoints (healthz, readyz). + #[arg(long, default_value_t = 8081, env = "OPENSHELL_HEALTH_PORT")] + health_port: u16, + /// Log level (trace, debug, info, warn, error). #[arg(long, default_value = "info", env = "OPENSHELL_LOG_LEVEL")] log_level: String, @@ -197,6 +201,14 @@ async fn run_from_args(args: Args) -> Result<()> { ); let bind = SocketAddr::from(([0, 0, 0, 0], args.port)); + let health_bind = SocketAddr::from(([0, 0, 0, 0], args.health_port)); + + if args.port == args.health_port { + return Err(miette::miette!( + "--port and --health-port must be different (both set to {})", + args.port + )); + } let tls = if args.disable_tls { None @@ -224,6 +236,7 @@ async fn run_from_args(args: Args) -> Result<()> { let mut config = openshell_core::Config::new(tls) .with_bind_address(bind) + .with_health_bind_address(health_bind) .with_log_level(&args.log_level); config = config diff --git a/crates/openshell-server/src/http.rs b/crates/openshell-server/src/http.rs index afe7edc1b6..4d45c53521 100644 --- a/crates/openshell-server/src/http.rs +++ b/crates/openshell-server/src/http.rs @@ -47,8 +47,7 @@ pub fn health_router() -> Router { /// Create the HTTP router. pub fn http_router(state: Arc) -> Router { - health_router() - .merge(crate::ssh_tunnel::router(state.clone())) + crate::ssh_tunnel::router(state.clone()) .merge(crate::ws_tunnel::router(state.clone())) .merge(crate::auth::router(state)) } diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 0894e53422..c2ff8322ee 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -182,6 +182,22 @@ pub async fn run_server( info!(address = %config.bind_address, "Server listening"); + // Bind the unauthenticated health endpoint on a separate port. + let health_listener = TcpListener::bind(config.health_bind_address) + .await + .map_err(|e| { + Error::transport(format!( + "failed to bind health port {}: {e}", + config.health_bind_address + )) + })?; + info!(address = %config.health_bind_address, "Health server listening"); + tokio::spawn(async move { + if let Err(e) = axum::serve(health_listener, health_router().into_make_service()).await { + error!("Health server error: {e}"); + } + }); + // Build TLS acceptor when TLS is configured; otherwise serve plaintext. let tls_acceptor = if let Some(tls) = &config.tls { Some(TlsAcceptor::from_files( diff --git a/deploy/docker/cluster-healthcheck.sh b/deploy/docker/cluster-healthcheck.sh index e2828c6e54..6766ca95ad 100644 --- a/deploy/docker/cluster-healthcheck.sh +++ b/deploy/docker/cluster-healthcheck.sh @@ -78,5 +78,7 @@ kubectl -n openshell get secret openshell-ssh-handshake >/dev/null 2>&1 || exit # iptables rules for NodePort routing. Without this check the health check # can pass before the port is routable, causing "Connection refused" on the # host-mapped port. +# Use /dev/tcp/127.0.0.1/30051' 2>/dev/null || exit 1 +timeout 2 bash -c 'exec 3<>/dev/tcp/127.0.0.1/30051 && exec 3>&-' 2>/dev/null || exit 1 diff --git a/deploy/helm/openshell/templates/statefulset.yaml b/deploy/helm/openshell/templates/statefulset.yaml index ed503a78e7..8262511344 100644 --- a/deploy/helm/openshell/templates/statefulset.yaml +++ b/deploy/helm/openshell/templates/statefulset.yaml @@ -49,6 +49,8 @@ spec: args: - --port - {{ .Values.service.port | quote }} + - --health-port + - {{ .Values.service.healthPort | quote }} - --log-level - {{ .Values.server.logLevel }} - --db-url @@ -113,22 +115,28 @@ spec: - name: grpc containerPort: {{ .Values.service.port }} protocol: TCP + - name: health + containerPort: {{ .Values.service.healthPort }} + protocol: TCP startupProbe: - tcpSocket: - port: grpc + httpGet: + path: /healthz + port: health periodSeconds: {{ .Values.probes.startup.periodSeconds }} timeoutSeconds: {{ .Values.probes.startup.timeoutSeconds }} failureThreshold: {{ .Values.probes.startup.failureThreshold }} livenessProbe: - tcpSocket: - port: grpc + httpGet: + path: /healthz + port: health initialDelaySeconds: {{ .Values.probes.liveness.initialDelaySeconds }} periodSeconds: {{ .Values.probes.liveness.periodSeconds }} timeoutSeconds: {{ .Values.probes.liveness.timeoutSeconds }} failureThreshold: {{ .Values.probes.liveness.failureThreshold }} readinessProbe: - tcpSocket: - port: grpc + httpGet: + path: /readyz + port: health initialDelaySeconds: {{ .Values.probes.readiness.initialDelaySeconds }} periodSeconds: {{ .Values.probes.readiness.periodSeconds }} timeoutSeconds: {{ .Values.probes.readiness.timeoutSeconds }} diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index d698e8120d..2b8cc4e6ff 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -37,6 +37,7 @@ service: type: NodePort port: 8080 nodePort: 30051 + healthPort: 8081 # Pod restart behavior and health probe tuning. podLifecycle: From 42c3cf635d64f8451e5d58923326d73891fff82f Mon Sep 17 00:00:00 2001 From: Seth Jennings Date: Tue, 21 Apr 2026 15:57:08 -0500 Subject: [PATCH 022/142] fix(k8s-driver): use dedicated kube client without read_timeout for watches (#907) The 30s read_timeout on the shared kube client was killing the long-lived watch streams during idle periods, causing a reconnect cycle every 30 seconds. Use a separate client with no read_timeout for watch_sandboxes so the streams stay open indefinitely. --- .../openshell-driver-kubernetes/src/driver.rs | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index f70054805b..444e0f55d3 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -102,6 +102,7 @@ const WORKSPACE_SENTINEL: &str = ".workspace-initialized"; #[derive(Clone)] pub struct KubernetesComputeDriver { client: Client, + watch_client: Client, config: KubernetesComputeConfig, } @@ -117,17 +118,30 @@ impl std::fmt::Debug for KubernetesComputeDriver { impl KubernetesComputeDriver { pub async fn new(config: KubernetesComputeConfig) -> Result { - let mut kube_config = match kube::Config::incluster() { + let base_config = match kube::Config::incluster() { Ok(c) => c, Err(_) => kube::Config::infer() .await .map_err(kube::Error::InferConfig)?, }; + + let mut kube_config = base_config.clone(); kube_config.connect_timeout = Some(Duration::from_secs(10)); kube_config.read_timeout = Some(Duration::from_secs(30)); kube_config.write_timeout = Some(Duration::from_secs(30)); let client = Client::try_from(kube_config)?; - Ok(Self { client, config }) + + let mut watch_kube_config = base_config; + watch_kube_config.connect_timeout = Some(Duration::from_secs(10)); + watch_kube_config.read_timeout = None; + watch_kube_config.write_timeout = Some(Duration::from_secs(30)); + let watch_client = Client::try_from(watch_kube_config)?; + + Ok(Self { + client, + watch_client, + config, + }) } pub async fn capabilities(&self) -> Result { @@ -155,6 +169,12 @@ impl KubernetesComputeDriver { self.config.ssh_handshake_skew_secs } + fn watch_api(&self) -> Api { + let gvk = GroupVersionKind::gvk(SANDBOX_GROUP, SANDBOX_VERSION, SANDBOX_KIND); + let resource = ApiResource::from_gvk(&gvk); + Api::namespaced_with(self.watch_client.clone(), &self.config.namespace, &resource) + } + fn api(&self) -> Api { let gvk = GroupVersionKind::gvk(SANDBOX_GROUP, SANDBOX_VERSION, SANDBOX_KIND); let resource = ApiResource::from_gvk(&gvk); @@ -392,8 +412,8 @@ impl KubernetesComputeDriver { pub async fn watch_sandboxes(&self) -> Result { let namespace = self.config.namespace.clone(); - let sandbox_api = self.api(); - let event_api: Api = Api::namespaced(self.client.clone(), &namespace); + let sandbox_api = self.watch_api(); + let event_api: Api = Api::namespaced(self.watch_client.clone(), &namespace); let mut sandbox_stream = watcher::watcher(sandbox_api, watcher::Config::default()).boxed(); let mut event_stream = watcher::watcher(event_api, watcher::Config::default()).boxed(); let (tx, rx) = mpsc::channel(256); From cbcc4b7ee0c1a00de28a02c1a33551be9fe7f128 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Wed, 22 Apr 2026 07:57:53 -0700 Subject: [PATCH 023/142] feat(server): allow disabling health check listener (#915) --- crates/openshell-core/src/config.rs | 28 ++++++++++++++++++++-------- crates/openshell-server/src/cli.rs | 24 +++++++++++++----------- crates/openshell-server/src/lib.rs | 25 ++++++++++++++----------- 3 files changed, 47 insertions(+), 30 deletions(-) diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index 3217a783d7..e572537391 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -58,8 +58,10 @@ pub struct Config { pub bind_address: SocketAddr, /// Address to bind the unauthenticated health endpoint to. - #[serde(default = "default_health_bind_address")] - pub health_bind_address: SocketAddr, + /// + /// When `None`, the dedicated health listener is disabled. + #[serde(default)] + pub health_bind_address: Option, /// Log level (trace, debug, info, warn, error). #[serde(default = "default_log_level")] @@ -180,7 +182,7 @@ impl Config { pub fn new(tls: Option) -> Self { Self { bind_address: default_bind_address(), - health_bind_address: default_health_bind_address(), + health_bind_address: None, log_level: default_log_level(), tls, database_url: String::new(), @@ -210,7 +212,7 @@ impl Config { #[must_use] pub const fn with_health_bind_address(mut self, addr: SocketAddr) -> Self { - self.health_bind_address = addr; + self.health_bind_address = Some(addr); self } @@ -327,10 +329,6 @@ fn default_bind_address() -> SocketAddr { "0.0.0.0:8080".parse().expect("valid default address") } -fn default_health_bind_address() -> SocketAddr { - "0.0.0.0:8081".parse().expect("valid default address") -} - fn default_log_level() -> String { "info".to_string() } @@ -370,6 +368,7 @@ const fn default_ssh_session_ttl_secs() -> u64 { #[cfg(test)] mod tests { use super::{ComputeDriverKind, Config}; + use std::net::SocketAddr; #[test] fn compute_driver_kind_parses_supported_values() { @@ -400,4 +399,17 @@ mod tests { vec![ComputeDriverKind::Kubernetes] ); } + + #[test] + fn config_new_disables_health_bind_by_default() { + let cfg = Config::new(None); + assert!(cfg.health_bind_address.is_none()); + } + + #[test] + fn config_with_health_bind_address_sets_address() { + let addr: SocketAddr = "0.0.0.0:9090".parse().expect("valid address"); + let cfg = Config::new(None).with_health_bind_address(addr); + assert_eq!(cfg.health_bind_address, Some(addr)); + } } diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index 0b64de803a..796068d216 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -24,7 +24,8 @@ struct Args { port: u16, /// Port for unauthenticated health endpoints (healthz, readyz). - #[arg(long, default_value_t = 8081, env = "OPENSHELL_HEALTH_PORT")] + /// Set to 0 to disable the dedicated health listener. + #[arg(long, default_value_t = 0, env = "OPENSHELL_HEALTH_PORT")] health_port: u16, /// Log level (trace, debug, info, warn, error). @@ -201,15 +202,6 @@ async fn run_from_args(args: Args) -> Result<()> { ); let bind = SocketAddr::from(([0, 0, 0, 0], args.port)); - let health_bind = SocketAddr::from(([0, 0, 0, 0], args.health_port)); - - if args.port == args.health_port { - return Err(miette::miette!( - "--port and --health-port must be different (both set to {})", - args.port - )); - } - let tls = if args.disable_tls { None } else { @@ -236,9 +228,19 @@ async fn run_from_args(args: Args) -> Result<()> { let mut config = openshell_core::Config::new(tls) .with_bind_address(bind) - .with_health_bind_address(health_bind) .with_log_level(&args.log_level); + if args.health_port != 0 { + if args.port == args.health_port { + return Err(miette::miette!( + "--port and --health-port must be different (both set to {})", + args.port + )); + } + let health_bind = SocketAddr::from(([0, 0, 0, 0], args.health_port)); + config = config.with_health_bind_address(health_bind); + } + config = config .with_database_url(args.db_url) .with_compute_drivers(args.drivers) diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index c2ff8322ee..9501ea3b2d 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -182,21 +182,24 @@ pub async fn run_server( info!(address = %config.bind_address, "Server listening"); - // Bind the unauthenticated health endpoint on a separate port. - let health_listener = TcpListener::bind(config.health_bind_address) - .await - .map_err(|e| { + // Bind the unauthenticated health endpoint on a separate port when configured. + if let Some(health_bind_address) = config.health_bind_address { + let health_listener = TcpListener::bind(health_bind_address).await.map_err(|e| { Error::transport(format!( "failed to bind health port {}: {e}", - config.health_bind_address + health_bind_address )) })?; - info!(address = %config.health_bind_address, "Health server listening"); - tokio::spawn(async move { - if let Err(e) = axum::serve(health_listener, health_router().into_make_service()).await { - error!("Health server error: {e}"); - } - }); + info!(address = %health_bind_address, "Health server listening"); + tokio::spawn(async move { + if let Err(e) = axum::serve(health_listener, health_router().into_make_service()).await + { + error!("Health server error: {e}"); + } + }); + } else { + info!("Health server disabled"); + } // Build TLS acceptor when TLS is configured; otherwise serve plaintext. let tls_acceptor = if let Some(tls) = &config.tls { From 78b685ed85868fba527eea9eb4e5fd7a536fa0a4 Mon Sep 17 00:00:00 2001 From: Trevor McKay Date: Wed, 22 Apr 2026 11:13:27 -0400 Subject: [PATCH 024/142] feat: add configurable timeout for image transfer to gateway containerd (#914) the transfer of very large sandbox images to containerd can timeout depending on the size of the image and the speed of the local host. Made-with: Cursor --- crates/openshell-bootstrap/src/build.rs | 5 ++++- crates/openshell-bootstrap/src/docker.rs | 18 ++++++++++++++++++ crates/openshell-bootstrap/src/lib.rs | 5 ++++- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/crates/openshell-bootstrap/src/build.rs b/crates/openshell-bootstrap/src/build.rs index eaa2213118..fb9b4a63db 100644 --- a/crates/openshell-bootstrap/src/build.rs +++ b/crates/openshell-bootstrap/src/build.rs @@ -46,7 +46,10 @@ pub async fn build_and_push_image( on_log(format!( "Pushing image {tag} into gateway \"{gateway_name}\"" )); - let local_docker = Docker::connect_with_local_defaults() + // Use the long-timeout Docker client so `docker save` of multi-GB images + // doesn't trip the 120s bollard default mid-stream. Override with + // OPENSHELL_DOCKER_TIMEOUT_SECS=. + let local_docker = crate::docker::connect_local_for_large_transfers() .into_diagnostic() .wrap_err("failed to connect to local Docker daemon")?; let container = container_name(gateway_name); diff --git a/crates/openshell-bootstrap/src/docker.rs b/crates/openshell-bootstrap/src/docker.rs index be086e534e..65482739f6 100644 --- a/crates/openshell-bootstrap/src/docker.rs +++ b/crates/openshell-bootstrap/src/docker.rs @@ -23,6 +23,24 @@ use std::collections::HashMap; const REGISTRY_NAMESPACE_DEFAULT: &str = "openshell"; +/// Default total HTTP timeout for Docker API calls that stream large payloads +/// (e.g. `docker save` used by `sandbox create --from`). Bollard's own +/// `connect_with_local_defaults()` ceiling is 120s, which is far too short for +/// multi-GB image exports — a 7 GB image on a laptop SSD takes ~4–5 minutes. +/// One hour is a safe upper bound; override with `OPENSHELL_DOCKER_TIMEOUT_SECS`. +pub(crate) const DEFAULT_LARGE_TRANSFER_TIMEOUT_SECS: u64 = 3600; + +/// Build a local-Docker client suitable for large streaming transfers. +/// Respects `OPENSHELL_DOCKER_TIMEOUT_SECS` (in seconds); falls back to +/// [`DEFAULT_LARGE_TRANSFER_TIMEOUT_SECS`] when unset or unparseable. +pub fn connect_local_for_large_transfers() -> std::result::Result { + let secs: u64 = std::env::var("OPENSHELL_DOCKER_TIMEOUT_SECS") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(DEFAULT_LARGE_TRANSFER_TIMEOUT_SECS); + Ok(Docker::connect_with_local_defaults()?.with_timeout(std::time::Duration::from_secs(secs))) +} + /// Resolve the raw GPU device-ID list, replacing the `"auto"` sentinel with a /// concrete device ID based on whether CDI is enabled on the daemon. /// diff --git a/crates/openshell-bootstrap/src/lib.rs b/crates/openshell-bootstrap/src/lib.rs index 71d223d662..53f659fc68 100644 --- a/crates/openshell-bootstrap/src/lib.rs +++ b/crates/openshell-bootstrap/src/lib.rs @@ -521,7 +521,10 @@ where .collect(); if !images.is_empty() { log("[status] Deploying components".to_string()); - let local_docker = Docker::connect_with_local_defaults().into_diagnostic()?; + // Long-timeout client: `docker save` of multi-GB component + // images streams past bollard's 120s default. See + // docker::connect_local_for_large_transfers(). + let local_docker = docker::connect_local_for_large_transfers().into_diagnostic()?; let container = container_name(&name); let on_log_ref = Arc::clone(&on_log); let mut push_log = move |msg: String| { From e28ca07863ba8e3283422b92fdcbc6564d54e3f2 Mon Sep 17 00:00:00 2001 From: "John T. Myers" <9696606+johntmyers@users.noreply.github.com> Date: Wed, 22 Apr 2026 12:18:24 -0700 Subject: [PATCH 025/142] fix(sandbox): preserve explicit read-only baseline paths (#910) Closes #905 Skip proxy baseline read-write enrichment when a path is already explicitly configured as read-only, add regression coverage for the proto and local policy code paths, and document the precedence rule in the sandbox architecture docs. Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- architecture/sandbox.md | 2 +- architecture/security-policy.md | 2 +- crates/openshell-sandbox/src/lib.rs | 112 +++++++++++++++++++++++----- 3 files changed, 94 insertions(+), 22 deletions(-) diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 2015702f82..151b844a5e 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -452,7 +452,7 @@ Kernel-level error behavior (e.g., Landlock ABI unavailable) depends on `Landloc - `BestEffort`: Log a warning and continue without filesystem isolation - `HardRequirement`: Return a fatal error, aborting the sandbox -**Baseline path filtering**: System-injected baseline paths (e.g., `/app`) are pre-filtered by `enrich_proto_baseline_paths()` / `enrich_sandbox_baseline_paths()` using `Path::exists()` before they reach Landlock. User-specified paths are not pre-filtered -- they are evaluated at Landlock apply time so misconfigurations surface as warnings or errors. +**Baseline path filtering**: System-injected baseline paths (e.g., `/app`) are pre-filtered by `enrich_proto_baseline_paths()` / `enrich_sandbox_baseline_paths()` using `Path::exists()` before they reach Landlock. If a baseline `read_write` path is already present in `read_only`, enrichment skips the promotion so explicit policy intent is preserved. User-specified paths are not pre-filtered -- they are evaluated at Landlock apply time so misconfigurations surface as warnings or errors. ### Seccomp syscall filtering diff --git a/architecture/security-policy.md b/architecture/security-policy.md index 82f0cc1dfa..39c337112d 100644 --- a/architecture/security-policy.md +++ b/architecture/security-policy.md @@ -393,7 +393,7 @@ Controls Landlock LSM compatibility behavior. **Static field** -- immutable afte **Per-path error handling**: `PathFd::new()` (which wraps `open(path, O_PATH | O_CLOEXEC)`) can fail for several reasons beyond path non-existence: `EACCES` (permission denied), `ELOOP` (symlink loop), `ENAMETOOLONG`, `ENOTDIR`. Each failure is classified with a human-readable reason in logs. In `best_effort` mode, the path is skipped and ruleset construction continues. In `hard_requirement` mode, the error is fatal. -**Baseline path filtering**: The enrichment functions (`enrich_proto_baseline_paths`, `enrich_sandbox_baseline_paths`) pre-filter system-injected baseline paths (e.g., `/app`) by checking `Path::exists()` before adding them to the policy. This prevents missing baseline paths from reaching Landlock at all. User-specified paths are not pre-filtered — they are evaluated at Landlock apply time so that misconfigurations surface as warnings (`best_effort`) or errors (`hard_requirement`). +**Baseline path filtering**: The enrichment functions (`enrich_proto_baseline_paths`, `enrich_sandbox_baseline_paths`) pre-filter system-injected baseline paths (e.g., `/app`) by checking `Path::exists()` before adding them to the policy. This prevents missing baseline paths from reaching Landlock at all. If a baseline `read_write` path is explicitly configured in `read_only`, enrichment skips the promotion and preserves the stricter policy intent. User-specified paths are not pre-filtered — they are evaluated at Landlock apply time so that misconfigurations surface as warnings (`best_effort`) or errors (`hard_requirement`). **Zero-rule safety check**: If all paths in the ruleset fail to open, `apply()` returns an error rather than calling `restrict_self()` on an empty ruleset. An empty Landlock ruleset with `restrict_self()` would block all filesystem access — the inverse of the intended degradation behavior. This error is caught by the outer `BestEffort` handler, which logs a warning and continues without Landlock. diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 49be7dd711..34ee80bb59 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -1347,17 +1347,18 @@ fn enrich_proto_baseline_paths(proto: &mut openshell_core::proto::SandboxPolicy) } } for path in &rw { - if !fs.read_write.iter().any(|p| p == path) { - if !std::path::Path::new(path).exists() { - debug!( - path, - "Baseline read-write path does not exist, skipping enrichment" - ); - continue; - } - fs.read_write.push(path.clone()); - modified = true; + if fs.read_only.iter().any(|p| p == path) || fs.read_write.iter().any(|p| p == path) { + continue; } + if !std::path::Path::new(path).exists() { + debug!( + path, + "Baseline read-write path does not exist, skipping enrichment" + ); + continue; + } + fs.read_write.push(path.clone()); + modified = true; } if modified { @@ -1401,17 +1402,18 @@ fn enrich_sandbox_baseline_paths(policy: &mut SandboxPolicy) { } for path in &rw { let p = std::path::PathBuf::from(path); - if !policy.filesystem.read_write.contains(&p) { - if !p.exists() { - debug!( - path, - "Baseline read-write path does not exist, skipping enrichment" - ); - continue; - } - policy.filesystem.read_write.push(p); - modified = true; + if policy.filesystem.read_only.contains(&p) || policy.filesystem.read_write.contains(&p) { + continue; } + if !p.exists() { + debug!( + path, + "Baseline read-write path does not exist, skipping enrichment" + ); + continue; + } + policy.filesystem.read_write.push(p); + modified = true; } if modified { @@ -1429,6 +1431,7 @@ fn enrich_sandbox_baseline_paths(policy: &mut SandboxPolicy) { #[cfg(test)] mod baseline_tests { use super::*; + use crate::policy::{FilesystemPolicy, LandlockPolicy, ProcessPolicy}; #[test] fn proc_not_in_both_read_only_and_read_write_when_gpu_present() { @@ -1493,6 +1496,75 @@ mod baseline_tests { ); } } + + #[test] + fn proto_enrichment_preserves_explicit_read_only_for_baseline_read_write_paths() { + let mut policy = openshell_policy::restrictive_default_policy(); + policy.filesystem = Some(openshell_core::proto::FilesystemPolicy { + read_only: vec!["/tmp".to_string()], + read_write: vec![], + include_workdir: false, + }); + policy.network_policies.insert( + "test".into(), + openshell_core::proto::NetworkPolicyRule { + name: "test-rule".into(), + endpoints: vec![openshell_core::proto::NetworkEndpoint { + host: "example.com".into(), + port: 443, + ..Default::default() + }], + ..Default::default() + }, + ); + + enrich_proto_baseline_paths(&mut policy); + + let filesystem = policy.filesystem.expect("filesystem policy"); + assert!( + filesystem.read_only.contains(&"/tmp".to_string()), + "explicit read_only baseline path should be preserved" + ); + assert!( + !filesystem.read_write.contains(&"/tmp".to_string()), + "baseline enrichment must not promote explicit read_only /tmp to read_write" + ); + } + + #[test] + fn local_enrichment_preserves_explicit_read_only_for_baseline_read_write_paths() { + let mut policy = SandboxPolicy { + version: 1, + filesystem: FilesystemPolicy { + read_only: vec![std::path::PathBuf::from("/tmp")], + read_write: vec![], + include_workdir: false, + }, + network: NetworkPolicy { + mode: NetworkMode::Proxy, + proxy: Some(ProxyPolicy { http_addr: None }), + }, + landlock: LandlockPolicy::default(), + process: ProcessPolicy::default(), + }; + + enrich_sandbox_baseline_paths(&mut policy); + + assert!( + policy + .filesystem + .read_only + .contains(&std::path::PathBuf::from("/tmp")), + "explicit read_only baseline path should be preserved" + ); + assert!( + !policy + .filesystem + .read_write + .contains(&std::path::PathBuf::from("/tmp")), + "baseline enrichment must not promote explicit read_only /tmp to read_write" + ); + } } /// Load sandbox policy from local files or gRPC. From f954e5927aea80a8349adcf9468d78edc2e46c1e Mon Sep 17 00:00:00 2001 From: "John T. Myers" <9696606+johntmyers@users.noreply.github.com> Date: Wed, 22 Apr 2026 12:37:32 -0700 Subject: [PATCH 026/142] fix(sandbox): resolve sandbox host aliases in SSRF checks (#912) Closes #879 Resolve policy hostnames against the sandbox's /etc/hosts before DNS so Kubernetes hostAliases are visible to the proxy while keeping the existing allowed_ips SSRF enforcement semantics intact. Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- architecture/sandbox.md | 2 +- architecture/security-policy.md | 23 +- crates/openshell-sandbox/src/proxy.rs | 572 ++++++++++++++++++-------- docs/security/best-practices.mdx | 4 +- 4 files changed, 424 insertions(+), 177 deletions(-) diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 151b844a5e..5104a6dccd 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -808,7 +808,7 @@ Every CONNECT request to a non-`inference.local` target produces an `info!()` lo ### SSRF protection (internal IP rejection) -After OPA allows a connection, the proxy resolves DNS and rejects any host that resolves to an internal IP address (loopback, RFC 1918 private, link-local, or IPv4-mapped IPv6 equivalents). This defense-in-depth measure prevents SSRF attacks where an allowed hostname is pointed at internal infrastructure. The check is implemented by `resolve_and_reject_internal()` which calls `tokio::net::lookup_host()` and validates every resolved address via `is_internal_ip()`. If any resolved IP is internal, the connection receives a `403 Forbidden` response and a warning is logged. See [SSRF Protection](security-policy.md#ssrf-protection-internal-ip-rejection) for the full list of blocked ranges. +After OPA allows a connection, the proxy resolves the host using the sandbox's `/etc/hosts` first on Linux (via `/proc//root/etc/hosts`, which picks up Kubernetes `hostAliases`), then falls back to DNS. It rejects any host that resolves to an internal IP address (loopback, RFC 1918 private, link-local, or IPv4-mapped IPv6 equivalents). This defense-in-depth measure prevents SSRF attacks where an allowed hostname is pointed at internal infrastructure. The check is implemented by `resolve_and_reject_internal()`, which validates every resolved address via `is_internal_ip()`. If any resolved IP is internal, the connection receives a `403 Forbidden` response and a warning is logged. `hostAliases` only affect name resolution — private destinations still need `allowed_ips`. See [SSRF Protection](security-policy.md#ssrf-protection-internal-ip-rejection) for the full list of blocked ranges. IP classification helpers (`is_always_blocked_ip`, `is_always_blocked_net`, `is_internal_ip`) are shared from `openshell_core::net`. The `parse_allowed_ips` function rejects entries overlapping always-blocked ranges (loopback, link-local, unspecified) at load time with a hard error, and `implicit_allowed_ips_for_ip_host` skips synthesis for always-blocked literal IP hosts. The mechanistic mapper filters proposals for always-blocked destinations to prevent infinite TUI notification loops. diff --git a/architecture/security-policy.md b/architecture/security-policy.md index 39c337112d..e3ec6abbc9 100644 --- a/architecture/security-policy.md +++ b/architecture/security-policy.md @@ -1104,9 +1104,9 @@ IP classification helpers live in `crates/openshell-core/src/net.rs` and are sha Runtime resolution and enforcement functions remain in `crates/openshell-sandbox/src/proxy.rs`: -- **`resolve_and_reject_internal(host, port) -> Result, String>`**: Default SSRF check. Resolves DNS via `tokio::net::lookup_host()`, then checks every resolved address against `is_internal_ip()`. If any address is internal, the entire connection is rejected. +- **`resolve_and_reject_internal(host, port, entrypoint_pid) -> Result, String>`**: Default SSRF check. Resolves the host using the sandbox's `/etc/hosts` first on Linux (via `/proc//root/etc/hosts`, which captures Kubernetes `hostAliases`), then falls back to `tokio::net::lookup_host()`. It checks every resolved address against `is_internal_ip()`. If any address is internal, the entire connection is rejected. -- **`resolve_and_check_allowed_ips(host, port, allowed_ips) -> Result, String>`**: Allowlist-based SSRF check. Resolves DNS, rejects any always-blocked IPs, then verifies every resolved address matches at least one entry in the `allowed_ips` list. +- **`resolve_and_check_allowed_ips(host, port, allowed_ips, entrypoint_pid) -> Result, String>`**: Allowlist-based SSRF check. Resolves the host using the sandbox's `/etc/hosts` first on Linux, rejects any always-blocked IPs, then verifies every resolved address matches at least one entry in the `allowed_ips` list. - **`parse_allowed_ips(raw) -> Result, String>`**: Parses CIDR/IP strings into typed `IpNet` values. **Rejects entries at load time** that overlap always-blocked ranges (loopback, link-local, unspecified) via `is_always_blocked_net`. Accepts both CIDR notation (`10.0.5.0/24`) and bare IPs (`10.0.5.20`, treated as `/32`). This prevents confusing UX where an entry is accepted in policy but silently denied at runtime. @@ -1162,6 +1162,8 @@ The `allowed_ips` field on a `NetworkEndpoint` enables controlled access to priv **Load-time validation**: `parse_allowed_ips` rejects entries that overlap always-blocked ranges with a hard error at policy load time. This catches misconfigurations early — an entry like `127.0.0.0/8` or `0.0.0.0/0` in `allowed_ips` would be silently un-enforceable at runtime, so it is rejected before the policy is applied. The same validation runs in both file mode (sandbox startup) and gRPC mode (live policy updates via `OpaEngine::reload_from_proto`). +**Sandbox `/etc/hosts` and `hostAliases`**: On Linux, the proxy consults the sandbox's `/etc/hosts` before falling back to DNS. This gives policy evaluation the same hostname-to-IP view that sandboxed tools see when Kubernetes `hostAliases` populate `/etc/hosts`. This is resolver input only. It does **not** synthesize `allowed_ips`, and it does not bypass the private-IP SSRF check. If `searxng.local` resolves to `192.168.1.105`, the request still needs `allowed_ips: ["192.168.1.105/32"]` to succeed. + **Implicit `allowed_ips` for IP hosts**: When a policy endpoint has a literal IP address as its host (e.g., `host: 10.0.5.20`), the proxy synthesizes an `allowed_ips` entry automatically via `implicit_allowed_ips_for_ip_host`. If the host is an always-blocked address (e.g., `127.0.0.1`, `169.254.169.254`, `0.0.0.0`), the function returns empty and logs a warning — no `allowed_ips` entry is synthesized, so the standard SSRF rejection applies. This supports three usage modes: @@ -1172,6 +1174,23 @@ This supports three usage modes: | **Host + allowlist** | `host` + `allowed_ips` | Domain must match `host` AND resolve to an IP in `allowed_ips` | | **Hostless allowlist** | `allowed_ips` only (no `host`) | Any domain allowed on the specified `port`, as long as it resolves to an IP in `allowed_ips` | +Example: + +```yaml +network_policies: + websearch: + name: websearch + endpoints: + - host: searxng.local + port: 8080 + allowed_ips: + - "192.168.1.105/32" + binaries: + - path: /usr/bin/curl +``` + +With a matching sandbox `/etc/hosts` entry such as `192.168.1.105 searxng.local`, this policy works. Without the `allowed_ips` entry, the request stays blocked because the resolved destination is private. + #### `allowed_ips` Format Entries can be: diff --git a/crates/openshell-sandbox/src/proxy.rs b/crates/openshell-sandbox/src/proxy.rs index e3d4b5c136..f23fbec33d 100644 --- a/crates/openshell-sandbox/src/proxy.rs +++ b/crates/openshell-sandbox/src/proxy.rs @@ -18,7 +18,7 @@ use openshell_ocsf::{ use std::net::{IpAddr, SocketAddr}; use std::path::PathBuf; use std::sync::Arc; -use std::sync::atomic::AtomicU32; +use std::sync::atomic::{AtomicU32, Ordering}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; use tokio::sync::mpsc; @@ -483,6 +483,8 @@ async fn handle_tcp_connection( return Ok(()); } + let sandbox_entrypoint_pid = entrypoint_pid.load(Ordering::Acquire); + // Query allowed_ips from the matched endpoint config (if any). // When present, the SSRF check validates resolved IPs against this // allowlist instead of blanket-blocking all private IPs. @@ -499,54 +501,60 @@ async fn handle_tcp_connection( // allowed_ips mode: validate resolved IPs against CIDR allowlist. // Loopback and link-local are still always blocked. match parse_allowed_ips(&raw_allowed_ips) { - Ok(nets) => match resolve_and_check_allowed_ips(&host, port, &nets).await { - Ok(addrs) => TcpStream::connect(addrs.as_slice()) + Ok(nets) => { + match resolve_and_check_allowed_ips(&host, port, &nets, sandbox_entrypoint_pid) .await - .into_diagnostic()?, - Err(reason) => { - { - let event = NetworkActivityBuilder::new(crate::ocsf_ctx()) - .activity(ActivityId::Open) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .src_endpoint_addr(peer_addr.ip(), peer_addr.port()) - .actor_process( - Process::from_bypass(&binary_str, &pid_str, &ancestors_str) - .with_cmd_line(&cmdline_str), - ) - .firewall_rule("-", "ssrf") - .message(format!( - "CONNECT blocked: allowed_ips check failed for {host_lc}:{port}" - )) - .status_detail(&reason) - .build(); - ocsf_emit!(event); + { + Ok(addrs) => TcpStream::connect(addrs.as_slice()) + .await + .into_diagnostic()?, + Err(reason) => { + { + let event = NetworkActivityBuilder::new(crate::ocsf_ctx()) + .activity(ActivityId::Open) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .dst_endpoint(Endpoint::from_domain(&host_lc, port)) + .src_endpoint_addr(peer_addr.ip(), peer_addr.port()) + .actor_process( + Process::from_bypass(&binary_str, &pid_str, &ancestors_str) + .with_cmd_line(&cmdline_str), + ) + .firewall_rule("-", "ssrf") + .message(format!( + "CONNECT blocked: allowed_ips check failed for {host_lc}:{port}" + )) + .status_detail(&reason) + .build(); + ocsf_emit!(event); + } + emit_denial( + &denial_tx, + &host_lc, + port, + &binary_str, + &decision, + &reason, + "ssrf", + ); + respond( + &mut client, + &build_json_error_response( + 403, + "Forbidden", + "ssrf_denied", + &format!( + "CONNECT {host_lc}:{port} blocked: allowed_ips check failed" + ), + ), + ) + .await?; + return Ok(()); } - emit_denial( - &denial_tx, - &host_lc, - port, - &binary_str, - &decision, - &reason, - "ssrf", - ); - respond( - &mut client, - &build_json_error_response( - 403, - "Forbidden", - "ssrf_denied", - &format!("CONNECT {host_lc}:{port} blocked: allowed_ips check failed"), - ), - ) - .await?; - return Ok(()); } - }, + } Err(reason) => { { let event = NetworkActivityBuilder::new(crate::ocsf_ctx()) @@ -593,7 +601,7 @@ async fn handle_tcp_connection( } } else { // Default: reject all internal IPs (loopback, RFC 1918, link-local). - match resolve_and_reject_internal(&host, port).await { + match resolve_and_reject_internal(&host, port, sandbox_entrypoint_pid).await { Ok(addrs) => TcpStream::connect(addrs.as_slice()) .await .into_diagnostic()?, @@ -1575,7 +1583,8 @@ fn query_tls_mode( /// — synthesizing an `allowed_ips` entry for them would be silently /// un-enforceable at runtime. fn implicit_allowed_ips_for_ip_host(host: &str) -> Vec { - if let Ok(ip) = host.parse::() { + let lookup_host = normalize_host_lookup_key(host); + if let Ok(ip) = lookup_host.parse::() { if is_always_blocked_ip(ip) { warn!( host, @@ -1585,32 +1594,137 @@ fn implicit_allowed_ips_for_ip_host(host: &str) -> Vec { ); return vec![]; } - vec![host.to_string()] + vec![lookup_host.to_string()] } else { vec![] } } -/// Resolve DNS for a host:port and reject if any resolved address is internal. -/// -/// Returns the resolved `SocketAddr` list on success. Returns an error string -/// if any resolved IP is in an internal range or if DNS resolution fails. -async fn resolve_and_reject_internal( +fn normalize_host_lookup_key(host: &str) -> &str { + host.strip_prefix('[') + .and_then(|trimmed| trimmed.strip_suffix(']')) + .unwrap_or(host) +} + +fn resolve_ip_literal(host: &str, port: u16) -> Option> { + normalize_host_lookup_key(host) + .parse::() + .ok() + .map(|ip| vec![SocketAddr::new(ip, port)]) +} + +#[cfg(any(target_os = "linux", test))] +fn parse_hosts_file_for_host(contents: &str, host: &str) -> Vec { + let lookup_host = normalize_host_lookup_key(host); + let mut addrs = Vec::new(); + + for raw_line in contents.lines() { + let line = raw_line.split('#').next().unwrap_or("").trim(); + if line.is_empty() { + continue; + } + + let mut fields = line.split_whitespace(); + let Some(ip_str) = fields.next() else { + continue; + }; + let Ok(ip) = ip_str.parse::() else { + continue; + }; + + if fields.any(|alias| alias.eq_ignore_ascii_case(lookup_host)) && !addrs.contains(&ip) { + addrs.push(ip); + } + } + + addrs +} + +#[cfg(any(target_os = "linux", test))] +fn resolve_from_hosts_file_contents(contents: &str, host: &str, port: u16) -> Vec { + parse_hosts_file_for_host(contents, host) + .into_iter() + .map(|ip| SocketAddr::new(ip, port)) + .collect() +} + +#[cfg(target_os = "linux")] +async fn resolve_from_sandbox_hosts( host: &str, port: u16, + entrypoint_pid: u32, +) -> Option> { + if entrypoint_pid == 0 { + return None; + } + + let hosts_path = format!("/proc/{entrypoint_pid}/root/etc/hosts"); + let contents = match tokio::fs::read_to_string(&hosts_path).await { + Ok(contents) => contents, + Err(error) => { + debug!( + pid = entrypoint_pid, + path = %hosts_path, + host, + "Falling back to DNS; failed to read sandbox hosts file: {error}" + ); + return None; + } + }; + + let addrs = resolve_from_hosts_file_contents(&contents, host, port); + if addrs.is_empty() { None } else { Some(addrs) } +} + +#[cfg(not(target_os = "linux"))] +async fn resolve_from_sandbox_hosts( + _host: &str, + _port: u16, + _entrypoint_pid: u32, +) -> Option> { + None +} + +async fn resolve_socket_addrs( + host: &str, + port: u16, + entrypoint_pid: u32, ) -> std::result::Result, String> { - let addrs: Vec = tokio::net::lookup_host((host, port)) + if let Some(addrs) = resolve_ip_literal(host, port) { + return Ok(addrs); + } + + if let Some(addrs) = resolve_from_sandbox_hosts(host, port, entrypoint_pid).await { + return Ok(addrs); + } + + let lookup_host = normalize_host_lookup_key(host); + let addrs: Vec = tokio::net::lookup_host((lookup_host, port)) .await - .map_err(|e| format!("DNS resolution failed for {host}:{port}: {e}"))? + .map_err(|e| format!("DNS resolution failed for {lookup_host}:{port}: {e}"))? .collect(); if addrs.is_empty() { return Err(format!( - "DNS resolution returned no addresses for {host}:{port}" + "DNS resolution returned no addresses for {lookup_host}:{port}" )); } - for addr in &addrs { + Ok(addrs) +} + +fn reject_internal_resolved_addrs( + host: &str, + addrs: &[SocketAddr], +) -> std::result::Result<(), String> { + if addrs.is_empty() { + return Err(format!( + "DNS resolution returned no addresses for {}", + normalize_host_lookup_key(host) + )); + } + + for addr in addrs { if is_internal_ip(addr.ip()) { return Err(format!( "{host} resolves to internal address {}, connection rejected", @@ -1619,29 +1733,19 @@ async fn resolve_and_reject_internal( } } - Ok(addrs) + Ok(()) } -/// Resolve DNS and validate resolved addresses against a CIDR/IP allowlist. -/// -/// Rejects loopback and link-local unconditionally. For all other resolved -/// addresses, checks that each one matches at least one entry in `allowed_ips`. -/// Entries can be CIDR notation ("10.0.5.0/24") or exact IPs ("10.0.5.20"). -/// -/// Returns the resolved `SocketAddr` list on success. -async fn resolve_and_check_allowed_ips( +fn validate_allowed_ips_for_resolved_addrs( host: &str, port: u16, + addrs: &[SocketAddr], allowed_ips: &[ipnet::IpNet], -) -> std::result::Result, String> { - let addrs: Vec = tokio::net::lookup_host((host, port)) - .await - .map_err(|e| format!("DNS resolution failed for {host}:{port}: {e}"))? - .collect(); - +) -> std::result::Result<(), String> { if addrs.is_empty() { return Err(format!( - "DNS resolution returned no addresses for {host}:{port}" + "DNS resolution returned no addresses for {}", + normalize_host_lookup_key(host) )); } @@ -1652,7 +1756,7 @@ async fn resolve_and_check_allowed_ips( )); } - for addr in &addrs { + for addr in addrs { // Always block loopback and link-local if is_always_blocked_ip(addr.ip()) { return Err(format!( @@ -1671,6 +1775,40 @@ async fn resolve_and_check_allowed_ips( } } + Ok(()) +} + +/// Resolve a host:port using sandbox `/etc/hosts` first (when available), then +/// reject if any resolved address is internal. +/// +/// Returns the resolved `SocketAddr` list on success. Returns an error string +/// if any resolved IP is in an internal range or if DNS resolution fails. +async fn resolve_and_reject_internal( + host: &str, + port: u16, + entrypoint_pid: u32, +) -> std::result::Result, String> { + let addrs = resolve_socket_addrs(host, port, entrypoint_pid).await?; + reject_internal_resolved_addrs(host, &addrs)?; + Ok(addrs) +} + +/// Resolve a host:port using sandbox `/etc/hosts` first (when available), then +/// validate resolved addresses against a CIDR/IP allowlist. +/// +/// Rejects loopback and link-local unconditionally. For all other resolved +/// addresses, checks that each one matches at least one entry in `allowed_ips`. +/// Entries can be CIDR notation ("10.0.5.0/24") or exact IPs ("10.0.5.20"). +/// +/// Returns the resolved `SocketAddr` list on success. +async fn resolve_and_check_allowed_ips( + host: &str, + port: u16, + allowed_ips: &[ipnet::IpNet], + entrypoint_pid: u32, +) -> std::result::Result, String> { + let addrs = resolve_socket_addrs(host, port, entrypoint_pid).await?; + validate_allowed_ips_for_resolved_addrs(host, port, &addrs, allowed_ips)?; Ok(addrs) } @@ -2186,6 +2324,7 @@ async fn handle_forward_proxy( } }; let policy_str = matched_policy.as_deref().unwrap_or("-"); + let sandbox_entrypoint_pid = entrypoint_pid.load(Ordering::Acquire); // 4b. If the endpoint has L7 config, evaluate the request against // L7 policy. The forward proxy handles exactly one request per @@ -2377,14 +2516,18 @@ async fn handle_forward_proxy( raw_allowed_ips = implicit_allowed_ips_for_ip_host(&host); } - let addrs = if !raw_allowed_ips.is_empty() { - // allowed_ips mode: validate resolved IPs against CIDR allowlist. - match parse_allowed_ips(&raw_allowed_ips) { - Ok(nets) => match resolve_and_check_allowed_ips(&host, port, &nets).await { - Ok(addrs) => addrs, - Err(reason) => { + let addrs = + if !raw_allowed_ips.is_empty() { + // allowed_ips mode: validate resolved IPs against CIDR allowlist. + match parse_allowed_ips(&raw_allowed_ips) { + Ok(nets) => { + match resolve_and_check_allowed_ips(&host, port, &nets, sandbox_entrypoint_pid) + .await { - let event = HttpActivityBuilder::new(crate::ocsf_ctx()) + Ok(addrs) => addrs, + Err(reason) => { + { + let event = HttpActivityBuilder::new(crate::ocsf_ctx()) .activity(ActivityId::Other) .action(ActionId::Denied) .disposition(DispositionId::Blocked) @@ -2406,18 +2549,18 @@ async fn handle_forward_proxy( )) .status_detail(&reason) .build(); - ocsf_emit!(event); - } - emit_denial_simple( - denial_tx, - &host_lc, - port, - &binary_str, - &decision, - &reason, - "ssrf", - ); - respond( + ocsf_emit!(event); + } + emit_denial_simple( + denial_tx, + &host_lc, + port, + &binary_str, + &decision, + &reason, + "ssrf", + ); + respond( client, &build_json_error_response( 403, @@ -2427,12 +2570,13 @@ async fn handle_forward_proxy( ), ) .await?; - return Ok(()); + return Ok(()); + } + } } - }, - Err(reason) => { - { - let event = HttpActivityBuilder::new(crate::ocsf_ctx()) + Err(reason) => { + { + let event = HttpActivityBuilder::new(crate::ocsf_ctx()) .activity(ActivityId::Other) .action(ActionId::Denied) .disposition(DispositionId::Blocked) @@ -2454,39 +2598,39 @@ async fn handle_forward_proxy( )) .status_detail(&reason) .build(); - ocsf_emit!(event); - } - emit_denial_simple( - denial_tx, - &host_lc, - port, - &binary_str, - &decision, - &reason, - "ssrf", - ); - respond( - client, - &build_json_error_response( - 403, - "Forbidden", - "ssrf_denied", - &format!( - "{method} {host_lc}:{port} blocked: invalid allowed_ips in policy" + ocsf_emit!(event); + } + emit_denial_simple( + denial_tx, + &host_lc, + port, + &binary_str, + &decision, + &reason, + "ssrf", + ); + respond( + client, + &build_json_error_response( + 403, + "Forbidden", + "ssrf_denied", + &format!( + "{method} {host_lc}:{port} blocked: invalid allowed_ips in policy" + ), ), - ), - ) - .await?; - return Ok(()); + ) + .await?; + return Ok(()); + } } - } - } else { - // No allowed_ips: reject internal IPs, allow public IPs through. - match resolve_and_reject_internal(&host, port).await { - Ok(addrs) => addrs, - Err(reason) => { - { - let event = HttpActivityBuilder::new(crate::ocsf_ctx()) + } else { + // No allowed_ips: reject internal IPs, allow public IPs through. + match resolve_and_reject_internal(&host, port, sandbox_entrypoint_pid).await { + Ok(addrs) => addrs, + Err(reason) => { + { + let event = HttpActivityBuilder::new(crate::ocsf_ctx()) .activity(ActivityId::Other) .action(ActionId::Denied) .disposition(DispositionId::Blocked) @@ -2508,31 +2652,31 @@ async fn handle_forward_proxy( )) .status_detail(&reason) .build(); - ocsf_emit!(event); + ocsf_emit!(event); + } + emit_denial_simple( + denial_tx, + &host_lc, + port, + &binary_str, + &decision, + &reason, + "ssrf", + ); + respond( + client, + &build_json_error_response( + 403, + "Forbidden", + "ssrf_denied", + &format!("{method} {host_lc}:{port} blocked: internal address"), + ), + ) + .await?; + return Ok(()); } - emit_denial_simple( - denial_tx, - &host_lc, - port, - &binary_str, - &decision, - &reason, - "ssrf", - ); - respond( - client, - &build_json_error_response( - 403, - "Forbidden", - "ssrf_denied", - &format!("{method} {host_lc}:{port} blocked: internal address"), - ), - ) - .await?; - return Ok(()); } - } - }; + }; // 6. Connect upstream let mut upstream = match TcpStream::connect(addrs.as_slice()).await { @@ -2687,7 +2831,7 @@ fn is_benign_relay_error(err: &miette::Report) -> bool { #[cfg(test)] mod tests { use super::*; - use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; // -- is_internal_ip: IPv4 -- @@ -2845,9 +2989,93 @@ mod tests { // -- resolve_and_reject_internal -- + #[test] + fn test_parse_hosts_file_for_host_handles_comments_invalid_rows_and_case() { + let contents = r#" + # comment + 192.168.1.105 searxng.local searxng + bad-ip ignored.local + 93.184.216.34 Example.Local # trailing comment + ::1 loopback.local + 192.168.1.105 searxng.local + "#; + + let result = parse_hosts_file_for_host(contents, "SEARXNG.LOCAL"); + assert_eq!(result, vec![IpAddr::V4(Ipv4Addr::new(192, 168, 1, 105))]); + + let public = parse_hosts_file_for_host(contents, "example.local"); + assert_eq!(public, vec![IpAddr::V4(Ipv4Addr::new(93, 184, 216, 34))]); + } + + #[test] + fn test_resolve_from_hosts_file_contents_requires_exact_alias_match() { + let contents = "192.168.1.105 searxng.local\n"; + + assert!( + resolve_from_hosts_file_contents(contents, "searxng", 8080).is_empty(), + "partial alias match should not resolve" + ); + + let result = resolve_from_hosts_file_contents(contents, "searxng.local", 8080); + assert_eq!( + result, + vec![SocketAddr::new( + IpAddr::V4(Ipv4Addr::new(192, 168, 1, 105)), + 8080 + )] + ); + } + + #[test] + fn test_resolve_from_hosts_file_contents_public_ip_passes_default_ssrf_check() { + let addrs = + resolve_from_hosts_file_contents("93.184.216.34 example.local\n", "example.local", 80); + assert!(reject_internal_resolved_addrs("example.local", &addrs).is_ok()); + } + + #[test] + fn test_resolve_from_hosts_file_contents_private_ip_requires_allowed_ips() { + let addrs = resolve_from_hosts_file_contents( + "192.168.1.105 searxng.local\n", + "searxng.local", + 8080, + ); + + let err = reject_internal_resolved_addrs("searxng.local", &addrs).unwrap_err(); + assert!( + err.contains("internal address"), + "expected private hosts-file resolution to remain blocked: {err}" + ); + + let nets = parse_allowed_ips(&["192.168.1.105/32".to_string()]).unwrap(); + assert!( + validate_allowed_ips_for_resolved_addrs("searxng.local", 8080, &addrs, &nets).is_ok() + ); + } + + #[test] + fn test_resolve_from_hosts_file_contents_always_blocked_ip_stays_blocked() { + let addrs = + resolve_from_hosts_file_contents("127.0.0.1 loopback.local\n", "loopback.local", 80); + let nets = vec!["127.0.0.0/8".parse::().unwrap()]; + let err = validate_allowed_ips_for_resolved_addrs("loopback.local", 80, &addrs, &nets) + .unwrap_err(); + assert!( + err.contains("always-blocked"), + "expected always-blocked hosts-file resolution to stay blocked: {err}" + ); + } + + #[test] + fn test_resolve_from_hosts_file_contents_returns_empty_without_match() { + let result = + resolve_from_hosts_file_contents("192.168.1.105 searxng.local\n", "missing.local", 80); + assert!(result.is_empty()); + } + #[tokio::test] async fn test_rejects_localhost_resolution() { - let result = resolve_and_reject_internal("localhost", 80).await; + let result = resolve_and_reject_internal("localhost", 80, 0).await; assert!(result.is_err()); let err = result.unwrap_err(); assert!( @@ -2858,7 +3086,7 @@ mod tests { #[tokio::test] async fn test_rejects_loopback_ip_literal() { - let result = resolve_and_reject_internal("127.0.0.1", 443).await; + let result = resolve_and_reject_internal("127.0.0.1", 443, 0).await; assert!(result.is_err()); let err = result.unwrap_err(); assert!( @@ -2869,7 +3097,7 @@ mod tests { #[tokio::test] async fn test_rejects_metadata_ip() { - let result = resolve_and_reject_internal("169.254.169.254", 80).await; + let result = resolve_and_reject_internal("169.254.169.254", 80, 0).await; assert!(result.is_err()); let err = result.unwrap_err(); assert!( @@ -2880,7 +3108,7 @@ mod tests { #[tokio::test] async fn test_dns_failure_returns_error() { - let result = resolve_and_reject_internal("this-host-does-not-exist.invalid", 80).await; + let result = resolve_and_reject_internal("this-host-does-not-exist.invalid", 80, 0).await; assert!(result.is_err()); let err = result.unwrap_err(); assert!( @@ -3222,7 +3450,7 @@ mod tests { async fn test_resolve_check_allowed_ips_blocks_loopback() { // Construct nets directly (parse_allowed_ips now rejects always-blocked). let nets = vec!["127.0.0.0/8".parse::().unwrap()]; - let result = resolve_and_check_allowed_ips("127.0.0.1", 80, &nets).await; + let result = resolve_and_check_allowed_ips("127.0.0.1", 80, &nets, 0).await; assert!(result.is_err()); let err = result.unwrap_err(); assert!( @@ -3235,7 +3463,7 @@ mod tests { async fn test_resolve_check_allowed_ips_blocks_metadata() { // Construct nets directly (parse_allowed_ips now rejects always-blocked). let nets = vec!["169.254.0.0/16".parse::().unwrap()]; - let result = resolve_and_check_allowed_ips("169.254.169.254", 80, &nets).await; + let result = resolve_and_check_allowed_ips("169.254.169.254", 80, &nets, 0).await; assert!(result.is_err()); let err = result.unwrap_err(); assert!( @@ -3248,7 +3476,7 @@ mod tests { async fn test_resolve_check_allowed_ips_blocks_unspecified() { // Construct nets directly (parse_allowed_ips now rejects always-blocked). let nets = vec!["0.0.0.0/0".parse::().unwrap()]; - let result = resolve_and_check_allowed_ips("0.0.0.0", 80, &nets).await; + let result = resolve_and_check_allowed_ips("0.0.0.0", 80, &nets, 0).await; assert!(result.is_err()); let err = result.unwrap_err(); assert!( @@ -3261,7 +3489,7 @@ mod tests { async fn test_resolve_check_allowed_ips_rejects_outside_allowlist() { // 8.8.8.8 resolves to a public IP which is NOT in 10.0.0.0/8 let nets = parse_allowed_ips(&["10.0.0.0/8".to_string()]).unwrap(); - let result = resolve_and_check_allowed_ips("dns.google", 443, &nets).await; + let result = resolve_and_check_allowed_ips("dns.google", 443, &nets, 0).await; assert!(result.is_err()); let err = result.unwrap_err(); assert!( @@ -3277,17 +3505,17 @@ mod tests { // Use a public CIDR (parse_allowed_ips now rejects 0.0.0.0/0). let nets = parse_allowed_ips(&["8.8.8.0/24".to_string()]).unwrap(); // K8s API server port - let result = resolve_and_check_allowed_ips("8.8.8.8", 6443, &nets).await; + let result = resolve_and_check_allowed_ips("8.8.8.8", 6443, &nets, 0).await; assert!(result.is_err()); assert!(result.unwrap_err().contains("blocked control-plane port")); // etcd client port - let result = resolve_and_check_allowed_ips("8.8.8.8", 2379, &nets).await; + let result = resolve_and_check_allowed_ips("8.8.8.8", 2379, &nets, 0).await; assert!(result.is_err()); assert!(result.unwrap_err().contains("blocked control-plane port")); // kubelet API port - let result = resolve_and_check_allowed_ips("8.8.8.8", 10250, &nets).await; + let result = resolve_and_check_allowed_ips("8.8.8.8", 10250, &nets, 0).await; assert!(result.is_err()); assert!(result.unwrap_err().contains("blocked control-plane port")); } @@ -3296,7 +3524,7 @@ mod tests { async fn test_resolve_check_allowed_ips_allows_non_control_plane_ports() { // Port 443 should not be blocked by the control-plane port list let nets = parse_allowed_ips(&["8.8.8.0/24".to_string()]).unwrap(); - let result = resolve_and_check_allowed_ips("8.8.8.8", 443, &nets).await; + let result = resolve_and_check_allowed_ips("8.8.8.8", 443, &nets, 0).await; assert!(result.is_ok()); } @@ -3624,7 +3852,7 @@ mod tests { async fn test_forward_public_ip_allowed_without_allowed_ips() { // Public IPs (e.g. dns.google -> 8.8.8.8) should pass through // resolve_and_reject_internal without needing allowed_ips. - let result = resolve_and_reject_internal("dns.google", 80).await; + let result = resolve_and_reject_internal("dns.google", 80, 0).await; assert!( result.is_ok(), "Public IP should be allowed without allowed_ips: {result:?}" @@ -3644,7 +3872,7 @@ mod tests { #[tokio::test] async fn test_forward_private_ip_rejected_without_allowed_ips() { // Private IP literals should be rejected by resolve_and_reject_internal. - let result = resolve_and_reject_internal("10.0.0.1", 80).await; + let result = resolve_and_reject_internal("10.0.0.1", 80, 0).await; assert!( result.is_err(), "Private IP should be rejected without allowed_ips" @@ -3660,7 +3888,7 @@ mod tests { async fn test_forward_private_ip_accepted_with_allowed_ips() { // Private IP with matching allowed_ips should pass through. let nets = parse_allowed_ips(&["10.0.0.0/8".to_string()]).unwrap(); - let result = resolve_and_check_allowed_ips("10.0.0.1", 80, &nets).await; + let result = resolve_and_check_allowed_ips("10.0.0.1", 80, &nets, 0).await; assert!( result.is_ok(), "Private IP with matching allowed_ips should be accepted: {result:?}" @@ -3671,7 +3899,7 @@ mod tests { async fn test_forward_private_ip_rejected_with_wrong_allowed_ips() { // Private IP not in allowed_ips should be rejected. let nets = parse_allowed_ips(&["192.168.0.0/16".to_string()]).unwrap(); - let result = resolve_and_check_allowed_ips("10.0.0.1", 80, &nets).await; + let result = resolve_and_check_allowed_ips("10.0.0.1", 80, &nets, 0).await; assert!( result.is_err(), "Private IP not in allowed_ips should be rejected" @@ -3688,7 +3916,7 @@ mod tests { // Loopback addresses are always blocked, even if in allowed_ips. // Construct nets directly (parse_allowed_ips now rejects always-blocked). let nets = vec!["127.0.0.0/8".parse::().unwrap()]; - let result = resolve_and_check_allowed_ips("127.0.0.1", 80, &nets).await; + let result = resolve_and_check_allowed_ips("127.0.0.1", 80, &nets, 0).await; assert!(result.is_err(), "Loopback should be always blocked"); let err = result.unwrap_err(); assert!( @@ -3702,7 +3930,7 @@ mod tests { // Link-local / cloud metadata addresses are always blocked. // Construct nets directly (parse_allowed_ips now rejects always-blocked). let nets = vec!["169.254.0.0/16".parse::().unwrap()]; - let result = resolve_and_check_allowed_ips("169.254.169.254", 80, &nets).await; + let result = resolve_and_check_allowed_ips("169.254.169.254", 80, &nets, 0).await; assert!(result.is_err(), "Link-local should be always blocked"); let err = result.unwrap_err(); assert!( diff --git a/docs/security/best-practices.mdx b/docs/security/best-practices.mdx index 87faae4a3f..6a618520f2 100644 --- a/docs/security/best-practices.mdx +++ b/docs/security/best-practices.mdx @@ -118,9 +118,9 @@ After OPA policy allows a connection, the proxy resolves DNS and rejects connect | Aspect | Detail | |---|---| | Default | The proxy blocks all private IPs. Loopback (`127.0.0.0/8`), link-local (`169.254.0.0/16`), and unspecified (`0.0.0.0`) addresses are always blocked and cannot be overridden with `allowed_ips`. | -| What you can change | Add `allowed_ips` (CIDR notation) to an endpoint to permit connections to specific private IP ranges. Policies with `allowed_ips` entries that overlap loopback, link-local, or unspecified addresses fail to load with a clear validation error. | +| What you can change | Add `allowed_ips` (CIDR notation) to an endpoint to permit connections to specific private IP ranges. On Linux, the proxy consults the sandbox's `/etc/hosts` before DNS, so Kubernetes `hostAliases` can make LAN-only hostnames resolvable. Policies with `allowed_ips` entries that overlap loopback, link-local, or unspecified addresses fail to load with a clear validation error. | | Risk if relaxed | Without SSRF protection, a misconfigured policy could allow the agent to reach cloud metadata services (`169.254.169.254`), internal databases, or other infrastructure endpoints through DNS rebinding. | -| Recommendation | Use `allowed_ips` only for known internal services. Scope the CIDR as narrowly as possible (for example, `10.0.5.20/32` for a single host). Loopback, link-local, and unspecified addresses are always blocked regardless of `allowed_ips`. The policy advisor does not propose rules for always-blocked destinations. | +| Recommendation | Use `allowed_ips` only for known internal services. Scope the CIDR as narrowly as possible (for example, `10.0.5.20/32` for a single host). Loopback, link-local, and unspecified addresses are always blocked regardless of `allowed_ips`. `hostAliases` change resolution, not authorization: `host: searxng.local` with `/etc/hosts` mapping to `192.168.1.105` still needs `allowed_ips: ["192.168.1.105/32"]`. The policy advisor does not propose rules for always-blocked destinations. | ### Operator Approval From 2f8e8ac3116fb3e6ee8657fc360e5562f426f7e3 Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Thu, 23 Apr 2026 04:35:21 +0800 Subject: [PATCH 027/142] fix(sandbox): inject GIT_SSL_CAINFO so git clone trusts the sandbox CA (#918) Ubuntu Noble's git links against libcurl-gnutls, which does not read SSL_CERT_FILE. Without GIT_SSL_CAINFO, `git clone` over HTTPS fails inside every community sandbox image with "server certificate verification failed". Add GIT_SSL_CAINFO to tls_env_vars pointing at the same combined CA bundle already used by CURL_CA_BUNDLE / REQUESTS_CA_BUNDLE. Fixes #790 Fixes NVIDIA/NemoClaw#2270 Fixes NVIDIA/NemoClaw#1828 Signed-off-by: Tinson Lai --- crates/openshell-sandbox/src/child_env.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/openshell-sandbox/src/child_env.rs b/crates/openshell-sandbox/src/child_env.rs index ebd47e2251..914e06ea5a 100644 --- a/crates/openshell-sandbox/src/child_env.rs +++ b/crates/openshell-sandbox/src/child_env.rs @@ -24,14 +24,17 @@ pub(crate) fn proxy_env_vars(proxy_url: &str) -> [(&'static str, String); 9] { pub(crate) fn tls_env_vars( ca_cert_path: &Path, combined_bundle_path: &Path, -) -> [(&'static str, String); 4] { +) -> [(&'static str, String); 5] { let ca_cert_path = ca_cert_path.display().to_string(); let combined_bundle_path = combined_bundle_path.display().to_string(); [ ("NODE_EXTRA_CA_CERTS", ca_cert_path.clone()), ("SSL_CERT_FILE", combined_bundle_path.clone()), ("REQUESTS_CA_BUNDLE", combined_bundle_path.clone()), - ("CURL_CA_BUNDLE", combined_bundle_path), + ("CURL_CA_BUNDLE", combined_bundle_path.clone()), + // Ubuntu Noble's git links against libcurl-gnutls, which ignores SSL_CERT_FILE. + // git reads GIT_SSL_CAINFO (or http.sslCAInfo) to locate the CA bundle. + ("GIT_SSL_CAINFO", combined_bundle_path), ] } @@ -79,5 +82,8 @@ mod tests { assert!(stdout.contains("NODE_EXTRA_CA_CERTS=/etc/openshell-tls/openshell-ca.pem")); assert!(stdout.contains("SSL_CERT_FILE=/etc/openshell-tls/ca-bundle.pem")); + assert!(stdout.contains("REQUESTS_CA_BUNDLE=/etc/openshell-tls/ca-bundle.pem")); + assert!(stdout.contains("CURL_CA_BUNDLE=/etc/openshell-tls/ca-bundle.pem")); + assert!(stdout.contains("GIT_SSL_CAINFO=/etc/openshell-tls/ca-bundle.pem")); } } From 30ddca42d3effe81792aa97e3f05e5b26f27651a Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Wed, 22 Apr 2026 14:15:10 -0700 Subject: [PATCH 028/142] ci(e2e): enable E2E to run on external forks throught the copy-pr-bot flow (#922) --- .github/actions/pr-gate/action.yml | 56 +++++++++++++++ .github/workflows/branch-e2e.yml | 43 +++++++++--- .github/workflows/e2e-gate-check.yml | 101 +++++++++++++++++++++++++++ .github/workflows/e2e-gate.yml | 41 +++++++++++ .github/workflows/test-gpu.yml | 55 +++++---------- 5 files changed, 250 insertions(+), 46 deletions(-) create mode 100644 .github/actions/pr-gate/action.yml create mode 100644 .github/workflows/e2e-gate-check.yml create mode 100644 .github/workflows/e2e-gate.yml diff --git a/.github/actions/pr-gate/action.yml b/.github/actions/pr-gate/action.yml new file mode 100644 index 0000000000..be1a5b19f8 --- /dev/null +++ b/.github/actions/pr-gate/action.yml @@ -0,0 +1,56 @@ +name: PR Gate +description: > + Resolve PR metadata for a `pull-request/` push from copy-pr-bot and decide + whether the workflow should run. Sets `should-run=true` only when the pushed + SHA still matches the PR head SHA and the PR carries the required label. For + non-`push` events (e.g. `workflow_dispatch`), always sets `should-run=true`. + +inputs: + required_label: + description: PR label required to enable the run (e.g. "test:e2e"). + required: true + +outputs: + should_run: + description: "true if the workflow should proceed, false otherwise" + value: ${{ steps.gate.outputs.should_run }} + +runs: + using: composite + steps: + - id: get_pr_info + if: github.event_name == 'push' + continue-on-error: true + uses: nv-gha-runners/get-pr-info@main + + - id: gate + shell: bash + env: + EVENT_NAME: ${{ github.event_name }} + GITHUB_SHA_VALUE: ${{ github.sha }} + GET_PR_INFO_OUTCOME: ${{ steps.get_pr_info.outcome }} + PR_INFO: ${{ steps.get_pr_info.outputs.pr-info }} + REQUIRED_LABEL: ${{ inputs.required_label }} + run: | + if [ "$EVENT_NAME" != "push" ]; then + echo "should_run=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if [ "$GET_PR_INFO_OUTCOME" != "success" ]; then + echo "should_run=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + head_sha="$(jq -r '.head.sha' <<< "$PR_INFO")" + has_label="$(jq -r --arg L "$REQUIRED_LABEL" '[.labels[].name] | index($L) != null' <<< "$PR_INFO")" + + # Only trust copied pull-request/* pushes that still match the PR head + # SHA and carry the required label. + if [ "$head_sha" = "$GITHUB_SHA_VALUE" ] && [ "$has_label" = "true" ]; then + should_run=true + else + should_run=false + fi + + echo "should_run=$should_run" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index ad53bb6358..1312c66b4a 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -1,16 +1,35 @@ name: Branch E2E Checks on: - pull_request: - types: [opened, synchronize, reopened, labeled] + push: + branches: + - "pull-request/[0-9]+" + workflow_dispatch: {} -permissions: - contents: read - packages: write +permissions: {} jobs: + pr_metadata: + name: Resolve PR metadata + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + outputs: + should_run: ${{ steps.gate.outputs.should_run }} + steps: + - uses: actions/checkout@v4 + - id: gate + uses: ./.github/actions/pr-gate + with: + required_label: test:e2e + build-gateway: - if: contains(github.event.pull_request.labels.*.name, 'test:e2e') + needs: [pr_metadata] + if: needs.pr_metadata.outputs.should_run == 'true' + permissions: + contents: read + packages: write uses: ./.github/workflows/docker-build.yml with: component: gateway @@ -18,7 +37,11 @@ jobs: runner: build-arm64 build-cluster: - if: contains(github.event.pull_request.labels.*.name, 'test:e2e') + needs: [pr_metadata] + if: needs.pr_metadata.outputs.should_run == 'true' + permissions: + contents: read + packages: write uses: ./.github/workflows/docker-build.yml with: component: cluster @@ -26,7 +49,11 @@ jobs: runner: build-arm64 e2e: - needs: [build-gateway, build-cluster] + needs: [pr_metadata, build-gateway, build-cluster] + if: needs.pr_metadata.outputs.should_run == 'true' + permissions: + contents: read + packages: read uses: ./.github/workflows/e2e-test.yml with: image-tag: ${{ github.sha }} diff --git a/.github/workflows/e2e-gate-check.yml b/.github/workflows/e2e-gate-check.yml new file mode 100644 index 0000000000..81b71ce64e --- /dev/null +++ b/.github/workflows/e2e-gate-check.yml @@ -0,0 +1,101 @@ +name: E2E Gate Check + +# Reusable gate that enforces: when `required_label` is present on a PR, +# `workflow_file` must have completed successfully for the PR head SHA. +# +# Callers wire their own triggers (`pull_request` + `workflow_run` for the +# workflow this gate guards) and pass in the label and workflow filename. + +on: + workflow_call: + inputs: + required_label: + description: PR label that makes the gated workflow mandatory. + required: true + type: string + workflow_file: + description: Filename of the workflow whose run must have succeeded (e.g. "branch-e2e.yml"). + required: true + type: string + +permissions: {} + +jobs: + check: + name: Enforce ${{ inputs.required_label }} runs when labeled + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + actions: read + steps: + - name: Resolve PR context + id: pr + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + EVENT_NAME: ${{ github.event_name }} + PR_HEAD_SHA_FROM_EVENT: ${{ github.event.pull_request.head.sha }} + PR_LABELS_FROM_EVENT: ${{ toJSON(github.event.pull_request.labels.*.name) }} + WORKFLOW_RUN_HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + shell: bash + run: | + set -euo pipefail + if [ "$EVENT_NAME" = "pull_request" ]; then + head_sha="$PR_HEAD_SHA_FROM_EVENT" + labels_json=$(jq -c . <<< "$PR_LABELS_FROM_EVENT") + else + head_sha="$WORKFLOW_RUN_HEAD_SHA" + pr=$(gh api "repos/$GH_REPO/commits/$head_sha/pulls" --jq '.[0] // empty') + if [ -z "$pr" ]; then + echo "No PR associated with $head_sha; gate is a no-op." + echo "skip=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + labels_json=$(jq -c '[.labels[].name]' <<< "$pr") + fi + echo "head_sha=$head_sha" >> "$GITHUB_OUTPUT" + echo "labels_json=$labels_json" >> "$GITHUB_OUTPUT" + + - name: Evaluate gate + if: steps.pr.outputs.skip != 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + HEAD_SHA: ${{ steps.pr.outputs.head_sha }} + LABELS_JSON: ${{ steps.pr.outputs.labels_json }} + REQUIRED_LABEL: ${{ inputs.required_label }} + WORKFLOW_FILE: ${{ inputs.workflow_file }} + shell: bash + run: | + set -euo pipefail + + has_label=$(jq -r --arg L "$REQUIRED_LABEL" 'any(.[]; . == $L)' <<< "$LABELS_JSON") + if [ "$has_label" != "true" ]; then + echo "::notice::$REQUIRED_LABEL not applied; gate passes." + exit 0 + fi + + runs=$(gh api "repos/$GH_REPO/actions/workflows/$WORKFLOW_FILE/runs?head_sha=$HEAD_SHA&event=push" --jq '.workflow_runs') + latest=$(jq -c 'sort_by(.created_at) | reverse | .[0] // empty' <<< "$runs") + + if [ -z "$latest" ]; then + echo "::error::$REQUIRED_LABEL is applied but $WORKFLOW_FILE has not run for $HEAD_SHA. Wait for copy-pr-bot to mirror the PR, or re-run the gate once the workflow completes." + exit 1 + fi + + status=$(jq -r '.status' <<< "$latest") + conclusion=$(jq -r '.conclusion' <<< "$latest") + + if [ "$conclusion" = "success" ]; then + echo "$WORKFLOW_FILE succeeded for $HEAD_SHA." + exit 0 + fi + + if [ "$status" != "completed" ]; then + echo "::error::$WORKFLOW_FILE is $status for $HEAD_SHA. This gate will re-evaluate on completion." + exit 1 + fi + + echo "::error::$WORKFLOW_FILE concluded as $conclusion for $HEAD_SHA." + exit 1 diff --git a/.github/workflows/e2e-gate.yml b/.github/workflows/e2e-gate.yml new file mode 100644 index 0000000000..757a954662 --- /dev/null +++ b/.github/workflows/e2e-gate.yml @@ -0,0 +1,41 @@ +name: E2E Gate + +on: + pull_request: + types: [opened, synchronize, reopened, labeled, unlabeled, ready_for_review] + workflow_run: + workflows: ["Branch E2E Checks", "GPU Test"] + types: [completed] + +permissions: {} + +jobs: + e2e: + name: E2E + # Run on every PR event; on workflow_run, only when the upstream was the + # matching gated workflow. + if: >- + github.event_name == 'pull_request' || + github.event.workflow_run.name == 'Branch E2E Checks' + permissions: + contents: read + pull-requests: read + actions: read + uses: ./.github/workflows/e2e-gate-check.yml + with: + required_label: test:e2e + workflow_file: branch-e2e.yml + + gpu: + name: GPU E2E + if: >- + github.event_name == 'pull_request' || + github.event.workflow_run.name == 'GPU Test' + permissions: + contents: read + pull-requests: read + actions: read + uses: ./.github/workflows/e2e-gate-check.yml + with: + required_label: test:e2e-gpu + workflow_file: test-gpu.yml diff --git a/.github/workflows/test-gpu.yml b/.github/workflows/test-gpu.yml index df953b5d31..dd406701f0 100644 --- a/.github/workflows/test-gpu.yml +++ b/.github/workflows/test-gpu.yml @@ -7,57 +7,30 @@ on: workflow_dispatch: {} # Add `schedule:` here when we want nightly coverage from the same workflow. -permissions: - contents: read - pull-requests: read - packages: write +permissions: {} jobs: pr_metadata: name: Resolve PR metadata runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read outputs: should_run: ${{ steps.gate.outputs.should_run }} steps: - - id: get_pr_info - if: github.event_name == 'push' - continue-on-error: true - uses: nv-gha-runners/get-pr-info@main - + - uses: actions/checkout@v4 - id: gate - shell: bash - env: - EVENT_NAME: ${{ github.event_name }} - GITHUB_SHA_VALUE: ${{ github.sha }} - GET_PR_INFO_OUTCOME: ${{ steps.get_pr_info.outcome }} - PR_INFO: ${{ steps.get_pr_info.outputs.pr-info }} - run: | - if [ "$EVENT_NAME" != "push" ]; then - echo "should_run=true" >> "$GITHUB_OUTPUT" - exit 0 - fi - - if [ "$GET_PR_INFO_OUTCOME" != "success" ]; then - echo "should_run=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - - head_sha="$(jq -r '.head.sha' <<< "$PR_INFO")" - has_gpu_label="$(jq -r '[.labels[].name] | index("test:e2e-gpu") != null' <<< "$PR_INFO")" - - # Only trust copied pull-request/* pushes that still match the PR head SHA - # and are explicitly labeled for GPU coverage. - if [ "$head_sha" = "$GITHUB_SHA_VALUE" ] && [ "$has_gpu_label" = "true" ]; then - should_run=true - else - should_run=false - fi - - echo "should_run=$should_run" >> "$GITHUB_OUTPUT" + uses: ./.github/actions/pr-gate + with: + required_label: test:e2e-gpu build-gateway: needs: [pr_metadata] if: needs.pr_metadata.outputs.should_run == 'true' + permissions: + contents: read + packages: write uses: ./.github/workflows/docker-build.yml with: component: gateway @@ -65,6 +38,9 @@ jobs: build-cluster: needs: [pr_metadata] if: needs.pr_metadata.outputs.should_run == 'true' + permissions: + contents: read + packages: write uses: ./.github/workflows/docker-build.yml with: component: cluster @@ -72,6 +48,9 @@ jobs: e2e-gpu: needs: [pr_metadata, build-gateway, build-cluster] if: needs.pr_metadata.outputs.should_run == 'true' + permissions: + contents: read + packages: read uses: ./.github/workflows/e2e-gpu-test.yaml with: image-tag: ${{ github.sha }} From 4483c860e0f2567880fa6dc8d0f90e0bcd7a5140 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Wed, 22 Apr 2026 16:42:35 -0700 Subject: [PATCH 029/142] feat(server,driver-vm,e2e): gateway-owned readiness + VM compute driver e2e (#901) --- Cargo.lock | 15 + architecture/custom-vm-runtime.md | 354 ++++++------- architecture/gateway.md | 2 +- crates/openshell-driver-vm/Cargo.toml | 8 + crates/openshell-driver-vm/Makefile | 7 - crates/openshell-driver-vm/README.md | 46 +- .../scripts/openshell-vm-sandbox-init.sh | 23 +- crates/openshell-driver-vm/src/driver.rs | 125 +++-- crates/openshell-driver-vm/src/ffi.rs | 3 - crates/openshell-driver-vm/src/lib.rs | 3 +- crates/openshell-driver-vm/src/main.rs | 46 +- crates/openshell-driver-vm/src/procguard.rs | 196 ++++++++ crates/openshell-driver-vm/src/runtime.rs | 464 ++++++------------ crates/openshell-driver-vm/start.sh | 26 +- crates/openshell-server/src/compute/mod.rs | 247 +++++++++- crates/openshell-server/src/lib.rs | 11 +- .../src/supervisor_session.rs | 29 ++ crates/openshell-vm/scripts/build-rootfs.sh | 63 +++ e2e/rust/e2e-vm.sh | 420 ++++++++-------- tasks/scripts/vm/smoke-orphan-cleanup.sh | 204 ++++++++ tasks/scripts/vm/vm-setup.sh | 2 +- tasks/test.toml | 3 +- tasks/vm.toml | 21 +- 23 files changed, 1497 insertions(+), 821 deletions(-) delete mode 100644 crates/openshell-driver-vm/Makefile create mode 100644 crates/openshell-driver-vm/src/procguard.rs create mode 100755 tasks/scripts/vm/smoke-orphan-cleanup.sh diff --git a/Cargo.lock b/Cargo.lock index d5de42fb36..2d0bc6ce22 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3101,6 +3101,7 @@ dependencies = [ "miette", "nix", "openshell-core", + "polling", "prost-types", "tar", "tokio", @@ -3672,6 +3673,20 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" +[[package]] +name = "polling" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi", + "pin-project-lite", + "rustix 1.1.4", + "windows-sys 0.61.2", +] + [[package]] name = "poly1305" version = "0.8.0" diff --git a/architecture/custom-vm-runtime.md b/architecture/custom-vm-runtime.md index 548b86d178..4cafe424fb 100644 --- a/architecture/custom-vm-runtime.md +++ b/architecture/custom-vm-runtime.md @@ -1,140 +1,161 @@ # Custom libkrunfw VM Runtime -> Status: Experimental and work in progress (WIP). VM support is under active development and may change. +> Status: Experimental and work in progress (WIP). The VM compute driver is +> under active development and may change. ## Overview -The OpenShell gateway VM uses [libkrun](https://github.com/containers/libkrun) to boot a -lightweight microVM with Apple Hypervisor.framework (macOS) or KVM (Linux). The kernel -is embedded inside `libkrunfw`, a companion library that packages a pre-built Linux kernel. +The OpenShell gateway uses [libkrun](https://github.com/containers/libkrun) via the +`openshell-driver-vm` compute driver to boot a lightweight microVM per sandbox. +Each VM runs on Apple Hypervisor.framework (macOS) or KVM (Linux), with the guest +kernel embedded inside `libkrunfw`. -The stock `libkrunfw` from Homebrew ships a minimal kernel without bridge, netfilter, or -conntrack support. This is insufficient for Kubernetes pod networking. +The stock `libkrunfw` from Homebrew ships a minimal kernel without bridge, +netfilter, or conntrack support. That is insufficient for the sandbox supervisor's +per-sandbox network namespace primitives (veth pair + iptables, see +`crates/openshell-sandbox/src/sandbox/linux/netns.rs`). The custom libkrunfw +runtime adds bridge, iptables/nftables, and conntrack support to the guest +kernel. -The custom libkrunfw runtime adds bridge CNI, iptables/nftables, and conntrack support to -the VM kernel, enabling standard Kubernetes networking. +The driver is spawned by `openshell-gateway` as a subprocess, talks to it over a +Unix domain socket (`compute-driver.sock`) with the +`openshell.compute.v1.ComputeDriver` gRPC surface, and manages per-sandbox +microVMs. The runtime (libkrun + libkrunfw + gvproxy) and the sandbox rootfs are +embedded directly in the driver binary — no sibling files required at runtime. ## Architecture ```mermaid graph TD subgraph Host["Host (macOS / Linux)"] - BIN[openshell-vm binary] - EMB["Embedded runtime (zstd-compressed)\nlibkrun · libkrunfw · gvproxy"] - CACHE["~/.local/share/openshell/vm-runtime/{version}/"] - PROV[Runtime provenance logging] - GVP[gvproxy networking proxy] - - BIN --> EMB - BIN -->|extracts to| CACHE - BIN --> PROV - BIN -->|spawns| GVP + GATEWAY["openshell-gateway
(compute::vm::spawn)"] + DRIVER["openshell-driver-vm
(compute-driver.sock)"] + EMB["Embedded runtime (zstd)
libkrun · libkrunfw · gvproxy
+ sandbox rootfs.tar.zst"] + GVP["gvproxy (per sandbox)
virtio-net · DHCP · DNS"] + + GATEWAY <-->|gRPC over UDS| DRIVER + DRIVER --> EMB + DRIVER -->|spawns one per sandbox| GVP end - subgraph Guest["Guest VM"] - INIT["openshell-vm-init.sh (PID 1)"] - VAL[Validates kernel capabilities] - CNI[Configures bridge CNI] - EXECA["Starts exec agent\nvsock port 10777"] - PKI[Generates mTLS PKI] - K3S[Execs k3s server] - EXECPY["openshell-vm-exec-agent.py"] - CHK["check-vm-capabilities.sh"] - - INIT --> VAL --> CNI --> EXECA --> PKI --> K3S + subgraph Guest["Per-sandbox microVM"] + SBXINIT["/srv/openshell-vm-sandbox-init.sh"] + SBX["/opt/openshell/bin/openshell-sandbox
(PID 1, supervisor)"] + SBXINIT --> SBX end - BIN -- "fork + krun_start_enter" --> INIT - GVP -- "virtio-net" --> Guest + DRIVER -- "fork + krun_start_enter" --> SBXINIT + GVP -- "virtio-net eth0" --> Guest + SBX -.->|"outbound ConnectSupervisor
gRPC stream"| GATEWAY + CLIENT["openshell-cli"] -->|SSH over supervisor relay| GATEWAY ``` +The driver spawns **one microVM per sandbox**. Each VM boots directly into +`openshell-sandbox` as PID 1. All gateway ingress — SSH, exec, connect — rides +the supervisor-initiated `ConnectSupervisor` gRPC stream opened from inside the +guest back out to the gateway, so gvproxy is configured with `-ssh-port -1` and +never binds a host-side TCP listener. + ## Embedded Runtime -The openshell-vm binary is fully self-contained, embedding both the VM runtime libraries -and a minimal rootfs as zstd-compressed byte arrays. On first use, the binary extracts -these to XDG cache directories with progress bars: +`openshell-driver-vm` embeds the VM runtime libraries and the sandbox rootfs as +zstd-compressed byte arrays, extracting on demand: ``` -~/.local/share/openshell/vm-runtime/{version}/ +~/.local/share/openshell/vm-runtime// # libkrun / libkrunfw / gvproxy ├── libkrun.{dylib,so} ├── libkrunfw.{5.dylib,so.5} └── gvproxy -~/.local/share/openshell/openshell-vm/{version}/instances//rootfs/ -├── usr/local/bin/k3s -├── opt/openshell/bin/openshell-sandbox -├── opt/openshell/manifests/ -└── ... +/sandboxes//rootfs/ # per-sandbox rootfs ``` -This eliminates the need for separate bundles or downloads - a single ~120MB binary -provides everything needed to run the VM. Old cache versions are automatically -cleaned up when a new version is extracted. +Old runtime cache versions are cleaned up when a new version is extracted. -### Hybrid Approach +### Sandbox rootfs preparation -The embedded rootfs uses a "minimal" configuration: -- Includes: Base Ubuntu, k3s binary, supervisor binary, helm charts, manifests -- Excludes: Pre-loaded container images (~1GB savings) +The rootfs tarball the driver embeds starts from the same minimal Ubuntu base +used across the project, and is **rewritten into a supervisor-only sandbox +guest** during extraction: -Container images are pulled on demand when sandboxes are created. First boot takes -~30-60s as k3s initializes; subsequent boots use cached state for ~3-5s startup. +- k3s state and Kubernetes manifests are stripped out +- `/srv/openshell-vm-sandbox-init.sh` is installed as the guest entrypoint +- the guest boots directly into `openshell-sandbox` — no k3s, no kube-proxy, + no CNI plugins -For the VM compute driver, the same embedded rootfs is rewritten into a -supervisor-only sandbox guest before boot: +See `crates/openshell-driver-vm/src/rootfs.rs` for the rewrite logic and +`crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh` for the init +script that gets installed. -- removes k3s state and Kubernetes manifests from the extracted rootfs -- installs `/srv/openshell-vm-sandbox-init.sh` -- boots directly into `openshell-sandbox` instead of `openshell-vm-init.sh` -- keeps the same embedded libkrun/libkrunfw kernel/runtime bundle +### `--internal-run-vm` helper -`openshell-driver-vm` now embeds the sandbox rootfs tarball independently so it can -prepare sandbox guests without linking against the `openshell-vm` Rust crate. -It now also embeds the minimal libkrun/libkrunfw bundle it needs for sandbox -boots and launches sandbox guests via a hidden helper mode in the -`openshell-driver-vm` binary itself, without depending on the `openshell-vm` -binary. The helper still starts its own embedded `gvproxy` instance to provide -virtio-net guest egress plus the single inbound SSH port forward used by the -compute driver. +The driver binary has two modes: the default mode is the gRPC server; when +launched with `--internal-run-vm` it becomes a per-sandbox launcher. The driver +spawns one launcher per sandbox as a subprocess, which in turn starts `gvproxy` +and calls `krun_start_enter` to boot the guest. Keeping the launcher in the +same binary means the driver ships a single artifact for both roles. -For fully air-gapped environments requiring pre-loaded images, build with: -```bash -mise run vm:rootfs # Full rootfs (~2GB, includes images) -mise run vm:build # Rebuild binary with full rootfs -``` +## Network Plane + +The driver launches a **dedicated `gvproxy` instance per sandbox** to provide the +guest's networking plane: + +- virtio-net backend over a Unix SOCK_STREAM (Linux) or SOCK_DGRAM (macOS vfkit) + socket, which surfaces as `eth0` inside the guest +- DHCP server + default router (192.168.127.1 / 192.168.127.2) for the guest's + udhcpc client +- DNS for host aliases: the guest init script seeds `/etc/hosts` with + `host.openshell.internal` → 192.168.127.1, while leaving gvproxy's legacy + `host.containers.internal` / `host.docker.internal` resolution intact + +The `-listen` API socket and the `-ssh-port` forwarder are both intentionally +omitted. After the supervisor-initiated relay migration the driver does not +enqueue any host-side port forwards, and the guest's SSH listener lives on a +Unix socket at `/run/openshell/ssh.sock` inside the VM that is reached over the +outbound `ConnectSupervisor` gRPC stream. Binding a host listener would race +concurrent sandboxes for port 2222 and surface a misleading "sshd is reachable" +endpoint. + +The sandbox supervisor's per-sandbox netns (veth pair + iptables) branches off +of this plane. libkrun's built-in TSI socket impersonation would not satisfy +those kernel-level primitives, which is why we need the custom libkrunfw. -## Network Profile +## Process Lifecycle Cleanup -The VM uses the bridge CNI profile, which requires a custom libkrunfw with bridge and -netfilter kernel support. The init script validates these capabilities at boot and fails -fast with an actionable error if they are missing. +`openshell-driver-vm` installs a cross-platform "die when my parent dies" +primitive (`procguard`) in every link of the spawn chain so that killing +`openshell-gateway` (SIGTERM, SIGKILL, or crash) reaps the driver, per-sandbox +launcher, gvproxy, and the libkrun worker: -### Bridge Profile +- Linux: `nix::sys::prctl::set_pdeathsig(SIGKILL)` +- macOS / BSDs: `smol-rs/polling` with `ProcessOps::Exit` on a helper thread +- gvproxy (the one non-Rust child) gets `PR_SET_PDEATHSIG` via `pre_exec` on + Linux, and is SIGTERM'd from the launcher's procguard cleanup callback on + macOS -- CNI: bridge plugin with `cni0` interface -- IP masquerade: enabled (iptables-legacy via CNI bridge plugin) -- kube-proxy: enabled (nftables mode) -- Service VIPs: functional (ClusterIP, NodePort) -- hostNetwork workarounds: not required +See `crates/openshell-driver-vm/src/procguard.rs` for the implementation and +`tasks/scripts/vm/smoke-orphan-cleanup.sh` (exposed as +`mise run vm:smoke:orphan-cleanup`) for the regression test that covers both +SIGTERM and SIGKILL paths. ## Runtime Provenance -At boot, the openshell-vm binary logs provenance metadata about the loaded runtime bundle: +At driver startup the loaded runtime bundle is logged with: - Library paths and SHA-256 hashes - Whether the runtime is custom-built or stock - For custom runtimes: libkrunfw commit, kernel version, build timestamp -This information is sourced from `provenance.json` (generated by the build script) -and makes it straightforward to correlate VM behavior with a specific runtime artifact. +This information is sourced from `provenance.json` (generated by the build +script) and makes it straightforward to correlate sandbox VM behavior with a +specific runtime artifact. ## Build Pipeline ```mermaid graph LR subgraph Source["crates/openshell-vm/runtime/"] - KCONF["kernel/openshell.kconfig\nKernel config fragment"] - README["README.md\nOperator documentation"] + KCONF["kernel/openshell.kconfig
Kernel config fragment"] end subgraph Linux["Linux CI (build-libkrun.sh)"] @@ -145,101 +166,87 @@ graph LR BUILD_M["Build libkrunfw.dylib + libkrun.dylib"] end - subgraph Output["target/libkrun-build/"] - LIB_SO["libkrunfw.so + libkrun.so\n(Linux)"] - LIB_DY["libkrunfw.dylib + libkrun.dylib\n(macOS)"] + subgraph Output["vm-runtime-<platform>.tar.zst"] + LIB_SO["libkrunfw.so + libkrun.so + gvproxy
(Linux)"] + LIB_DY["libkrunfw.dylib + libkrun.dylib + gvproxy
(macOS)"] end - KCONF --> BUILD_L - BUILD_L --> LIB_SO - KCONF --> BUILD_M - BUILD_M --> LIB_DY + KCONF --> BUILD_L --> LIB_SO + KCONF --> BUILD_M --> LIB_DY ``` +The `vm-runtime-.tar.zst` artifact is consumed by +`openshell-driver-vm`'s `build.rs`, which embeds the library set into the +binary via `include_bytes!()`. Setting `OPENSHELL_VM_RUNTIME_COMPRESSED_DIR` +at build time (wired up by `crates/openshell-driver-vm/start.sh`) points the +build at the staged artifacts. + ## Kernel Config Fragment -The `openshell.kconfig` fragment enables these kernel features on top of the stock -libkrunfw kernel: +The `openshell.kconfig` fragment enables these kernel features on top of the +stock libkrunfw kernel: | Feature | Key Configs | Purpose | |---------|-------------|---------| -| Network namespaces | `CONFIG_NET_NS`, `CONFIG_NAMESPACES` | Pod isolation | -| veth | `CONFIG_VETH` | Pod network namespace pairs | -| Bridge device | `CONFIG_BRIDGE`, `CONFIG_BRIDGE_NETFILTER` | cni0 bridge for pod networking, kube-proxy bridge traffic visibility | +| Network namespaces | `CONFIG_NET_NS`, `CONFIG_NAMESPACES` | Sandbox netns isolation | +| veth | `CONFIG_VETH` | Sandbox network namespace pairs | +| Bridge device | `CONFIG_BRIDGE`, `CONFIG_BRIDGE_NETFILTER` | Bridge support + iptables visibility into bridge traffic | | Netfilter framework | `CONFIG_NETFILTER`, `CONFIG_NETFILTER_ADVANCED`, `CONFIG_NETFILTER_XTABLES` | iptables/nftables framework | -| xtables match modules | `CONFIG_NETFILTER_XT_MATCH_CONNTRACK`, `_COMMENT`, `_MULTIPORT`, `_MARK`, `_STATISTIC`, `_ADDRTYPE`, `_RECENT`, `_LIMIT` | kube-proxy and kubelet iptables rules | +| xtables match modules | `CONFIG_NETFILTER_XT_MATCH_CONNTRACK`, `_COMMENT`, `_MULTIPORT`, `_MARK`, `_STATISTIC`, `_ADDRTYPE`, `_RECENT`, `_LIMIT` | Sandbox supervisor iptables rules | | Connection tracking | `CONFIG_NF_CONNTRACK`, `CONFIG_NF_CT_NETLINK` | NAT state tracking | -| NAT | `CONFIG_NF_NAT` | Service VIP DNAT/SNAT | -| iptables | `CONFIG_IP_NF_IPTABLES`, `CONFIG_IP_NF_FILTER`, `CONFIG_IP_NF_NAT`, `CONFIG_IP_NF_MANGLE` | CNI bridge masquerade and compat | -| nftables | `CONFIG_NF_TABLES`, `CONFIG_NFT_CT`, `CONFIG_NFT_NAT`, `CONFIG_NFT_MASQ`, `CONFIG_NFT_NUMGEN`, `CONFIG_NFT_FIB_IPV4` | kube-proxy nftables mode (primary) | -| IP forwarding | `CONFIG_IP_ADVANCED_ROUTER`, `CONFIG_IP_MULTIPLE_TABLES` | Pod-to-pod routing | -| IPVS | `CONFIG_IP_VS`, `CONFIG_IP_VS_RR`, `CONFIG_IP_VS_NFCT` | kube-proxy IPVS mode (optional) | -| Traffic control | `CONFIG_NET_SCH_HTB`, `CONFIG_NET_CLS_CGROUP` | Kubernetes QoS | -| Cgroups | `CONFIG_CGROUPS`, `CONFIG_CGROUP_DEVICE`, `CONFIG_MEMCG`, `CONFIG_CGROUP_PIDS` | Container resource limits | -| TUN/TAP | `CONFIG_TUN` | CNI plugin support | +| NAT | `CONFIG_NF_NAT` | Sandbox egress DNAT/SNAT | +| iptables | `CONFIG_IP_NF_IPTABLES`, `CONFIG_IP_NF_FILTER`, `CONFIG_IP_NF_NAT`, `CONFIG_IP_NF_MANGLE` | Masquerade and compat | +| nftables | `CONFIG_NF_TABLES`, `CONFIG_NFT_CT`, `CONFIG_NFT_NAT`, `CONFIG_NFT_MASQ`, `CONFIG_NFT_NUMGEN`, `CONFIG_NFT_FIB_IPV4` | nftables path | +| IP forwarding | `CONFIG_IP_ADVANCED_ROUTER`, `CONFIG_IP_MULTIPLE_TABLES` | Sandbox-to-host routing | +| Traffic control | `CONFIG_NET_SCH_HTB`, `CONFIG_NET_CLS_CGROUP` | QoS | +| Cgroups | `CONFIG_CGROUPS`, `CONFIG_CGROUP_DEVICE`, `CONFIG_MEMCG`, `CONFIG_CGROUP_PIDS` | Sandbox resource limits | +| TUN/TAP | `CONFIG_TUN` | CNI plugin compatibility; inherited from the shared kconfig, not exercised by the driver. | | Dummy interface | `CONFIG_DUMMY` | Fallback networking | -| Landlock | `CONFIG_SECURITY_LANDLOCK` | Filesystem sandboxing support | -| Seccomp filter | `CONFIG_SECCOMP_FILTER` | Syscall filtering support | +| Landlock | `CONFIG_SECURITY_LANDLOCK` | Sandbox supervisor filesystem sandboxing | +| Seccomp filter | `CONFIG_SECCOMP_FILTER` | Sandbox supervisor syscall filtering | -See `crates/openshell-vm/runtime/kernel/openshell.kconfig` for the full fragment with -inline comments explaining why each option is needed. +See `crates/openshell-vm/runtime/kernel/openshell.kconfig` for the full +fragment with inline comments explaining why each option is needed. ## Verification -One verification tool is provided: - -1. **Capability checker** (`check-vm-capabilities.sh`): Runs inside the VM to verify - kernel capabilities. Produces pass/fail results for each required feature. - -## Running Commands In A Live VM - -The standalone `openshell-vm` binary supports `openshell-vm exec -- ` for a running VM. - -- Each VM instance stores local runtime state next to its instance rootfs -- libkrun maps a per-instance host Unix socket into the guest on vsock port `10777` -- `openshell-vm-init.sh` starts `openshell-vm-exec-agent.py` during boot -- `openshell-vm exec` connects to the host socket, which libkrun forwards into the guest exec agent -- The guest exec agent spawns the command, then streams stdout, stderr, and exit status back -- The host-side bootstrap also uses the exec agent to read PKI cert files from the guest - (via `cat /opt/openshell/pki/`) instead of requiring a separate vsock server - -`openshell-vm exec` also injects `KUBECONFIG=/etc/rancher/k3s/k3s.yaml` by default so kubectl-style -commands work the same way they would inside the VM shell. +- **Capability checker** (`check-vm-capabilities.sh`): runs inside a sandbox VM + to verify kernel capabilities. Produces pass/fail results for each required + feature. +- **Orphan-cleanup smoke test**: `mise run vm:smoke:orphan-cleanup` asserts + that killing the gateway leaves zero driver, launcher, gvproxy, or libkrun + survivors. ## Build Commands -```bash +```shell # One-time setup: download pre-built runtime (~30s) mise run vm:setup -# Build and run -mise run vm - -# Build embedded binary with base rootfs (~120MB, recommended) -mise run vm:rootfs -- --base # Build base rootfs tarball -mise run vm:build # Build binary with embedded rootfs - -# Build with full rootfs (air-gapped, ~2GB+) -mise run vm:rootfs # Build full rootfs tarball -mise run vm:build # Rebuild binary +# Start openshell-gateway with the VM compute driver +mise run gateway:vm # With custom kernel (optional, adds ~20 min) -FROM_SOURCE=1 mise run vm:setup # Build runtime from source -mise run vm:build # Then build embedded binary +FROM_SOURCE=1 mise run vm:setup # Wipe everything and start over mise run vm:clean ``` +See `crates/openshell-driver-vm/README.md` for the full driver workflow, +including multi-gateway development, CLI registration, and sandbox creation +examples. + ## CI/CD -The openshell-vm build is split into two GitHub Actions workflows that publish to a -rolling `vm-dev` GitHub Release: +Two GitHub Actions workflows back the driver's release artifacts, both +publishing to a rolling `vm-dev` GitHub Release: ### Kernel Runtime (`release-vm-kernel.yml`) -Builds the custom libkrunfw (kernel firmware), libkrun (VMM), and gvproxy for all -supported platforms. Runs on-demand or when the kernel config / pinned versions change. +Builds the custom libkrunfw (kernel firmware), libkrun (VMM), and gvproxy for +all supported platforms. Runs on-demand or when the kernel config / pinned +versions change. | Platform | Runner | Build Method | |----------|--------|-------------| @@ -247,43 +254,36 @@ supported platforms. Runs on-demand or when the kernel config / pinned versions | Linux x86_64 | `build-amd64` (self-hosted) | Native `build-libkrun.sh` | | macOS ARM64 | `macos-latest-xlarge` (GitHub-hosted) | `build-libkrun-macos.sh` | -Artifacts: `vm-runtime-{platform}.tar.zst` containing libkrun, libkrunfw, gvproxy, and -provenance metadata. - -Each platform builds its own libkrunfw and libkrun natively. The kernel inside -libkrunfw is always Linux regardless of host platform. +Artifacts: `vm-runtime-{platform}.tar.zst` containing libkrun, libkrunfw, +gvproxy, and provenance metadata. Each platform builds its own libkrunfw and +libkrun natively; the kernel inside libkrunfw is always Linux regardless of +host platform. -### VM Binary (`release-vm-dev.yml`) +### Driver Binary (`release-vm-dev.yml`) -Builds the self-extracting openshell-vm binary for all platforms. Runs on every push -to `main` that touches VM-related crates. +Builds the self-contained `openshell-driver-vm` binary for every platform, +with the kernel runtime + sandbox rootfs embedded. Runs on every push to +`main` that touches VM-related crates. -```mermaid -graph TD - CV[compute-versions] --> DL[download-kernel-runtime\nfrom vm-dev release] - DL --> RFS_ARM[build-rootfs arm64] - DL --> RFS_AMD[build-rootfs amd64] - RFS_ARM --> VM_ARM[build-vm linux-arm64] - RFS_AMD --> VM_AMD[build-vm linux-amd64] - RFS_ARM --> VM_MAC["build-vm-macos\n(osxcross, reuses arm64 rootfs)"] - VM_ARM --> REL[release-vm-dev\nupload to rolling release] - VM_AMD --> REL - VM_MAC --> REL -``` +The `download-kernel-runtime` job pulls the current `vm-runtime-.tar.zst` +from the `vm-dev` release; the `build-openshell-driver-vm` jobs set +`OPENSHELL_VM_RUNTIME_COMPRESSED_DIR=$PWD/target/vm-runtime-compressed` and +run `cargo build --release -p openshell-driver-vm`. The macOS driver is +cross-compiled via osxcross (no macOS runner needed for the binary build — +only for the kernel build). -The macOS binary is cross-compiled via osxcross (no macOS runner needed for the binary -build — only for the kernel build). The macOS VM guest is always Linux ARM64, so it -reuses the arm64 rootfs. - -macOS binaries produced via osxcross are not codesigned. Users must self-sign: -```bash -codesign --entitlements crates/openshell-vm/entitlements.plist --force -s - ./openshell-vm -``` +macOS driver binaries produced via osxcross are not codesigned. Development +builds are signed automatically by `crates/openshell-driver-vm/start.sh`; a +packaged release needs signing in CI. ## Rollout Strategy -1. Custom runtime is embedded by default when building with `mise run vm:build`. -2. The init script validates kernel capabilities at boot and fails fast if missing. -3. For development, override with `OPENSHELL_VM_RUNTIME_DIR` to use a local directory. -4. In CI, kernel runtime is pre-built and cached in the `vm-dev` release. The binary - build downloads it via `download-kernel-runtime.sh`. +1. Custom runtime is embedded by default when building `openshell-driver-vm` + with `OPENSHELL_VM_RUNTIME_COMPRESSED_DIR` set (wired up by + `crates/openshell-driver-vm/start.sh`). +2. The sandbox init script validates kernel capabilities at boot and fails + fast if missing. +3. For development, override with `OPENSHELL_VM_RUNTIME_DIR` to use a local + directory instead of the extracted cache. +4. In CI, the kernel runtime is pre-built and cached in the `vm-dev` release. + The driver build downloads it via `download-kernel-runtime.sh`. diff --git a/architecture/gateway.md b/architecture/gateway.md index 5dd2419af7..9e9da67859 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -605,7 +605,7 @@ The gateway reaches the sandbox exclusively through the supervisor-initiated `Co - **Create**: The VM driver process allocates a sandbox-specific rootfs from its own embedded `rootfs.tar.zst`, injects an explicitly configured guest mTLS bundle when the gateway callback endpoint is `https://`, then re-execs itself in a hidden helper mode that loads libkrun directly and boots the supervisor. - **Networking**: The helper starts an embedded `gvproxy`, wires it into libkrun as virtio-net, and gives the guest outbound connectivity. No inbound TCP listener is needed — the supervisor reaches the gateway over its outbound `ConnectSupervisor` stream. -- **Gateway callback**: The guest init script configures `eth0` for gvproxy networking, prefers the configured `OPENSHELL_GRPC_ENDPOINT`, and falls back to host aliases or the gvproxy gateway IP (`192.168.127.1`) when local hostname resolution is unavailable on macOS. +- **Gateway callback**: The guest init script configures `eth0` for gvproxy networking, seeds `/etc/hosts` so `host.openshell.internal` resolves to the gvproxy gateway IP (`192.168.127.1`), preserves gvproxy's legacy `host.containers.internal` / `host.docker.internal` DNS answers, prefers the configured `OPENSHELL_GRPC_ENDPOINT`, and falls back to those aliases or the raw gateway IP when local hostname resolution is unavailable on macOS. - **Guest boot**: The sandbox guest runs a minimal init script that starts `openshell-sandbox` directly as PID 1 inside the VM. - **Watch stream**: Emits provisioning, ready, error, deleting, deleted, and platform-event updates so the gateway store remains the durable source of truth. diff --git a/crates/openshell-driver-vm/Cargo.toml b/crates/openshell-driver-vm/Cargo.toml index 368716ef96..b4d92b0fc1 100644 --- a/crates/openshell-driver-vm/Cargo.toml +++ b/crates/openshell-driver-vm/Cargo.toml @@ -37,5 +37,13 @@ libloading = "0.8" tar = "0.4" zstd = "0.13" +# smol-rs/polling drives the BSD/macOS parent-death detection in +# procguard via kqueue's EVFILT_PROC / NOTE_EXIT filter. We could use +# it on Linux too (via epoll + pidfd) but sticking with +# nix::sys::prctl::set_pdeathsig there keeps the Linux path a single +# syscall with no helper thread. +[target.'cfg(any(target_os = "macos", target_os = "ios", target_os = "freebsd", target_os = "netbsd", target_os = "openbsd", target_os = "dragonfly"))'.dependencies] +polling = "3.11" + [lints] workspace = true diff --git a/crates/openshell-driver-vm/Makefile b/crates/openshell-driver-vm/Makefile deleted file mode 100644 index e1c360f3da..0000000000 --- a/crates/openshell-driver-vm/Makefile +++ /dev/null @@ -1,7 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -.PHONY: start - -start: - ./start.sh diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index a95462695c..8808b25d90 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -31,19 +31,15 @@ Sandbox guests execute `/opt/openshell/bin/openshell-sandbox` as PID 1 inside th ## Quick start (recommended) -`start.sh` handles runtime setup, builds, codesigning, and environment wiring. From the repo root: ```shell -crates/openshell-driver-vm/start.sh +mise run gateway:vm ``` -or equivalently: - -```shell -make -C crates/openshell-driver-vm start -``` -First run takes a few minutes while `mise run vm:setup` stages libkrun/libkrunfw/gvproxy and `mise run vm:rootfs -- --base` builds the embedded rootfs. Subsequent runs are cached. State lives under `target/openshell-vm-driver-dev/` (SQLite DB + per-sandbox rootfs + `compute-driver.sock`). +First run takes a few minutes while `mise run vm:setup` stages libkrun/libkrunfw/gvproxy and `mise run vm:rootfs -- --base` builds the embedded rootfs. Subsequent runs are cached. To keep the Unix socket path under macOS `SUN_LEN`, `mise run gateway:vm` and `start.sh` default the state dir to `/tmp/openshell-vm-driver-dev-$USER-port-$PORT/` (SQLite DB + per-sandbox rootfs + `compute-driver.sock`) unless `OPENSHELL_VM_DRIVER_STATE_DIR` is set. +The wrapper also prints the recommended gateway name (`vm-driver-port-$PORT` by default) plus the exact repo-local `scripts/bin/openshell gateway add` and `scripts/bin/openshell gateway select` commands to use from another terminal. This avoids accidentally hitting an older `openshell` binary elsewhere on your `PATH`. +It also exports `OPENSHELL_DRIVER_DIR=$PWD/target/debug` before starting the gateway so local dev runs use the freshly built `openshell-driver-vm` instead of an older installed copy from `~/.local/libexec/openshell` or `/usr/local/libexec`. Override via environment: @@ -53,10 +49,33 @@ OPENSHELL_SSH_HANDSHAKE_SECRET=$(openssl rand -hex 32) \ crates/openshell-driver-vm/start.sh ``` +Run multiple dev gateways side by side by giving each one a unique port. The wrapper derives a distinct default state dir from that port automatically: + +```shell +OPENSHELL_SERVER_PORT=8080 mise run gateway:vm +OPENSHELL_SERVER_PORT=8081 mise run gateway:vm +``` + +If you want a custom suffix instead of `port-$PORT`, set `OPENSHELL_VM_INSTANCE`: + +```shell +OPENSHELL_SERVER_PORT=8082 \ +OPENSHELL_VM_INSTANCE=feature-a \ +mise run gateway:vm +``` + +If you want a custom CLI gateway name, set `OPENSHELL_VM_GATEWAY_NAME`: + +```shell +OPENSHELL_SERVER_PORT=8082 \ +OPENSHELL_VM_GATEWAY_NAME=vm-feature-a \ +mise run gateway:vm +``` + Teardown: ```shell -rm -rf target/openshell-vm-driver-dev +rm -rf /tmp/openshell-vm-driver-dev-$USER-port-8080 ``` ## Manual equivalent @@ -78,16 +97,17 @@ codesign \ --force -s - target/debug/openshell-driver-vm # 4. Start the gateway with the VM driver -mkdir -p target/openshell-vm-driver-dev +mkdir -p /tmp/openshell-vm-driver-dev-$USER-port-8080 target/debug/openshell-gateway \ --drivers vm \ --disable-tls \ - --database-url sqlite:target/openshell-vm-driver-dev/openshell.db \ + --database-url sqlite:/tmp/openshell-vm-driver-dev-$USER-port-8080/openshell.db \ + --driver-dir $PWD/target/debug \ --grpc-endpoint http://host.containers.internal:8080 \ --ssh-handshake-secret dev-vm-driver-secret \ --ssh-gateway-host 127.0.0.1 \ --ssh-gateway-port 8080 \ - --vm-driver-state-dir $PWD/target/openshell-vm-driver-dev + --vm-driver-state-dir /tmp/openshell-vm-driver-dev-$USER-port-8080 ``` The gateway resolves `openshell-driver-vm` in this order: `--driver-dir`, conventional install locations (`~/.local/libexec/openshell`, `/usr/local/libexec/openshell`, `/usr/local/libexec`), then a sibling of the gateway binary. @@ -97,7 +117,7 @@ The gateway resolves `openshell-driver-vm` in this order: `--driver-dir`, conven | Flag | Env var | Default | Purpose | |---|---|---|---| | `--drivers vm` | `OPENSHELL_DRIVERS` | `kubernetes` | Select the VM compute driver. | -| `--grpc-endpoint URL` | `OPENSHELL_GRPC_ENDPOINT` | — | Required. URL the sandbox guest calls back to. Use a host alias that resolves to the gateway's host from inside the VM (gvproxy answers `host.containers.internal` and `host.openshell.internal` to `192.168.127.1`). | +| `--grpc-endpoint URL` | `OPENSHELL_GRPC_ENDPOINT` | — | Required. URL the sandbox guest calls back to. Use a host alias that resolves to the gateway's host from inside the VM (`host.containers.internal` comes from gvproxy DNS; the guest init script also seeds `host.openshell.internal` to `192.168.127.1`). | | `--vm-driver-state-dir DIR` | `OPENSHELL_VM_DRIVER_STATE_DIR` | `target/openshell-vm-driver` | Per-sandbox rootfs, console logs, and the `compute-driver.sock` UDS. | | `--driver-dir DIR` | `OPENSHELL_DRIVER_DIR` | unset | Override the directory searched for `openshell-driver-vm`. | | `--vm-driver-vcpus N` | `OPENSHELL_VM_DRIVER_VCPUS` | `2` | vCPUs per sandbox. | diff --git a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh index 70dda5acbc..e449003f97 100644 --- a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh +++ b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh @@ -9,6 +9,7 @@ set -euo pipefail BOOT_START=$(date +%s%3N 2>/dev/null || date +%s) +GVPROXY_GATEWAY_IP="192.168.127.1" ts() { local now @@ -72,6 +73,20 @@ tcp_probe() { fi } +ensure_host_gateway_aliases() { + local hosts_tmp="/tmp/openshell-hosts.$$" + + if [ -f /etc/hosts ]; then + grep -vE '(^|[[:space:]])host\.openshell\.internal([[:space:]]|$)' /etc/hosts > "$hosts_tmp" || true + else + : > "$hosts_tmp" + fi + + printf '%s host.openshell.internal\n' "$GVPROXY_GATEWAY_IP" >> "$hosts_tmp" + cat "$hosts_tmp" > /etc/hosts + rm -f "$hosts_tmp" +} + rewrite_openshell_endpoint_if_needed() { local endpoint="${OPENSHELL_ENDPOINT:-}" [ -n "$endpoint" ] || return 0 @@ -92,7 +107,7 @@ rewrite_openshell_endpoint_if_needed() { return 0 fi - for candidate in host.containers.internal host.docker.internal 192.168.127.1; do + for candidate in host.openshell.internal host.containers.internal host.docker.internal "$GVPROXY_GATEWAY_IP"; do if [ "$candidate" = "$host" ]; then continue fi @@ -163,18 +178,20 @@ DHCP_SCRIPT if ! udhcpc -i eth0 -f -q -n -T 1 -t 3 -A 1 -s "$UDHCPC_SCRIPT" 2>&1; then ts "WARNING: DHCP failed, falling back to static config" ip addr add 192.168.127.2/24 dev eth0 2>/dev/null || true - ip route add default via 192.168.127.1 2>/dev/null || true + ip route add default via "$GVPROXY_GATEWAY_IP" 2>/dev/null || true fi else ts "no DHCP client, using static config" ip addr add 192.168.127.2/24 dev eth0 2>/dev/null || true - ip route add default via 192.168.127.1 2>/dev/null || true + ip route add default via "$GVPROXY_GATEWAY_IP" 2>/dev/null || true fi if [ ! -s /etc/resolv.conf ]; then echo "nameserver 8.8.8.8" > /etc/resolv.conf echo "nameserver 8.8.4.4" >> /etc/resolv.conf fi + + ensure_host_gateway_aliases else ts "WARNING: eth0 not found; supervisor will start without guest egress" fi diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 8237ba03c1..d649a585a4 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -33,6 +33,8 @@ const DRIVER_NAME: &str = "openshell-driver-vm"; const WATCH_BUFFER: usize = 256; const DEFAULT_VCPUS: u8 = 2; const DEFAULT_MEM_MIB: u32 = 2048; +const GVPROXY_GATEWAY_IP: &str = "192.168.127.1"; +const OPENSHELL_HOST_GATEWAY_ALIAS: &str = "host.openshell.internal"; const GUEST_SSH_SOCKET_PATH: &str = "/run/openshell/ssh.sock"; const GUEST_TLS_DIR: &str = "/opt/openshell/tls"; const GUEST_TLS_CA_PATH: &str = "/opt/openshell/tls/ca.crt"; @@ -147,7 +149,7 @@ fn validate_openshell_endpoint(endpoint: &str) -> Result<(), String> { if invalid_from_vm { return Err(format!( - "openshell endpoint '{endpoint}' is not reachable from sandbox VMs; use a concrete host such as 127.0.0.1, host.containers.internal, or another routable address" + "openshell endpoint '{endpoint}' is not reachable from sandbox VMs; use a concrete host such as 127.0.0.1, {OPENSHELL_HOST_GATEWAY_ALIAS}, or another routable address" )); } @@ -256,7 +258,19 @@ impl VmDriver { let console_output = state_dir.join("rootfs-console.log"); let mut command = Command::new(&self.launcher_bin); - command.kill_on_drop(true); + // Intentionally DO NOT set kill_on_drop(true). On a signal-driven + // driver exit (SIGKILL, SIGTERM without a handler, panic), + // tokio's Drop is racy with the launcher's procguard-initiated + // cleanup: if kill_on_drop SIGKILLs the launcher first, its + // cleanup callback never gets to SIGTERM gvproxy, and gvproxy is + // reparented to init as an orphan. Instead the whole cleanup + // cascade runs via procguard: + // driver exits → launcher's kqueue (macOS) or PR_SET_PDEATHSIG + // (Linux) fires → launcher kills gvproxy + libkrun fork → + // launcher exits → its own children die under pdeathsig. + // The explicit Drop path in VmProcess::terminate_vm_process still + // handles voluntary `delete_sandbox` teardown cleanly, where we + // do want SIGTERM + wait + SIGKILL semantics. command.stdin(Stdio::null()); command.stdout(Stdio::inherit()); command.stderr(Stdio::inherit()); @@ -403,16 +417,23 @@ impl VmDriver { snapshots } + /// Watch the launcher child process and surface errors as driver + /// conditions. + /// + /// The driver no longer owns the `Ready` transition — the gateway + /// promotes a sandbox to `Ready` the moment its supervisor session + /// lands (see `openshell-server/src/compute/mod.rs`). This loop only + /// handles the sad paths: the child process failing to start, exiting + /// abnormally, or becoming unpollable. Those still surface as driver + /// `Error` conditions so the gateway can reason about a dead VM. async fn monitor_sandbox(&self, sandbox_id: String) { - let mut ready_emitted = false; - loop { - let (process, state_dir) = { + let process = { let registry = self.registry.lock().await; let Some(record) = registry.get(&sandbox_id) else { return; }; - (record.process.clone(), record.state_dir.clone()) + record.process.clone() }; let exit_status = { @@ -469,16 +490,6 @@ impl VmDriver { return; } - if !ready_emitted && guest_ssh_ready(&state_dir).await { - if let Some(snapshot) = self - .set_snapshot_condition(&sandbox_id, ready_condition(), false) - .await - { - self.publish_snapshot(snapshot); - } - ready_emitted = true; - } - tokio::time::sleep(Duration::from_millis(250)).await; } } @@ -726,7 +737,7 @@ fn guest_visible_openshell_endpoint(endpoint: &str) -> String { None => false, }; - if should_rewrite && url.set_host(Some("192.168.127.1")).is_ok() { + if should_rewrite && url.set_host(Some(GVPROXY_GATEWAY_IP)).is_ok() { return url.to_string(); } @@ -843,16 +854,6 @@ async fn terminate_vm_process(child: &mut Child) -> Result<(), std::io::Error> { } } -async fn guest_ssh_ready(state_dir: &Path) -> bool { - let console_log = state_dir.join("rootfs-console.log"); - let Ok(contents) = tokio::fs::read_to_string(console_log).await else { - return false; - }; - - contents.contains("SSH server is ready to accept connections") - || contents.contains("SSH server listening") -} - fn sandbox_snapshot(sandbox: &Sandbox, condition: SandboxCondition, deleting: bool) -> Sandbox { Sandbox { id: sandbox.id.clone(), @@ -895,16 +896,6 @@ fn provisioning_condition() -> SandboxCondition { } } -fn ready_condition() -> SandboxCondition { - SandboxCondition { - r#type: "Ready".to_string(), - status: "True".to_string(), - reason: "Listening".to_string(), - message: "Supervisor is listening for SSH connections".to_string(), - last_transition_time: String::new(), - } -} - fn deleting_condition() -> SandboxCondition { SandboxCondition { r#type: "Ready".to_string(), @@ -1030,19 +1021,47 @@ mod tests { let env = build_guest_environment(&sandbox, &config); assert!(env.contains(&"HOME=/root".to_string())); - assert!(env.contains(&"OPENSHELL_ENDPOINT=http://192.168.127.1:8080/".to_string())); + assert!(env.contains(&format!( + "OPENSHELL_ENDPOINT=http://{GVPROXY_GATEWAY_IP}:8080/" + ))); assert!(env.contains(&"OPENSHELL_SANDBOX_ID=sandbox-123".to_string())); assert!(env.contains(&format!( "OPENSHELL_SSH_SOCKET_PATH={GUEST_SSH_SOCKET_PATH}" ))); } + #[test] + fn guest_visible_openshell_endpoint_rewrites_loopback_hosts_to_gvproxy_gateway() { + assert_eq!( + guest_visible_openshell_endpoint("http://127.0.0.1:8080"), + format!("http://{GVPROXY_GATEWAY_IP}:8080/") + ); + assert_eq!( + guest_visible_openshell_endpoint("http://localhost:8080"), + format!("http://{GVPROXY_GATEWAY_IP}:8080/") + ); + assert_eq!( + guest_visible_openshell_endpoint("https://[::1]:8443"), + format!("https://{GVPROXY_GATEWAY_IP}:8443/") + ); + } + #[test] fn guest_visible_openshell_endpoint_preserves_non_loopback_hosts() { + assert_eq!( + guest_visible_openshell_endpoint(&format!( + "http://{OPENSHELL_HOST_GATEWAY_ALIAS}:8080" + )), + format!("http://{OPENSHELL_HOST_GATEWAY_ALIAS}:8080") + ); assert_eq!( guest_visible_openshell_endpoint("http://host.containers.internal:8080"), "http://host.containers.internal:8080" ); + assert_eq!( + guest_visible_openshell_endpoint(&format!("http://{GVPROXY_GATEWAY_IP}:8080")), + format!("http://{GVPROXY_GATEWAY_IP}:8080") + ); assert_eq!( guest_visible_openshell_endpoint("https://gateway.internal:8443"), "https://gateway.internal:8443" @@ -1157,9 +1176,9 @@ mod tests { fn validate_openshell_endpoint_accepts_host_gateway() { validate_openshell_endpoint("http://host.containers.internal:8080") .expect("guest-reachable host alias should be accepted"); - validate_openshell_endpoint("http://192.168.127.1:8080") + validate_openshell_endpoint(&format!("http://{GVPROXY_GATEWAY_IP}:8080")) .expect("gateway IP should be accepted"); - validate_openshell_endpoint("http://host.openshell.internal:8080") + validate_openshell_endpoint(&format!("http://{OPENSHELL_HOST_GATEWAY_ALIAS}:8080")) .expect("openshell host alias should be accepted"); validate_openshell_endpoint("https://gateway.internal:8443") .expect("dns endpoint should be accepted"); @@ -1214,32 +1233,6 @@ mod tests { let _ = std::fs::remove_dir_all(base); } - #[tokio::test] - async fn guest_ssh_ready_detects_guest_console_marker() { - let base = unique_temp_dir(); - std::fs::create_dir_all(&base).unwrap(); - std::fs::write( - base.join("rootfs-console.log"), - "...\nINFO openshell_sandbox: SSH server is ready to accept connections\n", - ) - .unwrap(); - - assert!(guest_ssh_ready(&base).await); - - let _ = std::fs::remove_dir_all(base); - } - - #[tokio::test] - async fn guest_ssh_ready_is_false_without_marker() { - let base = unique_temp_dir(); - std::fs::create_dir_all(&base).unwrap(); - std::fs::write(base.join("rootfs-console.log"), "sandbox booting\n").unwrap(); - - assert!(!guest_ssh_ready(&base).await); - - let _ = std::fs::remove_dir_all(base); - } - fn unique_temp_dir() -> PathBuf { static COUNTER: AtomicU64 = AtomicU64::new(0); let nanos = SystemTime::now() diff --git a/crates/openshell-driver-vm/src/ffi.rs b/crates/openshell-driver-vm/src/ffi.rs index 750788ac13..a81b150afb 100644 --- a/crates/openshell-driver-vm/src/ffi.rs +++ b/crates/openshell-driver-vm/src/ffi.rs @@ -37,7 +37,6 @@ type KrunSetExec = unsafe extern "C" fn( argv: *const *const c_char, envp: *const *const c_char, ) -> i32; -type KrunSetPortMap = unsafe extern "C" fn(ctx_id: u32, port_map: *const *const c_char) -> i32; type KrunSetConsoleOutput = unsafe extern "C" fn(ctx_id: u32, filepath: *const c_char) -> i32; type KrunStartEnter = unsafe extern "C" fn(ctx_id: u32) -> i32; type KrunDisableImplicitVsock = unsafe extern "C" fn(ctx_id: u32) -> i32; @@ -68,7 +67,6 @@ pub struct LibKrun { pub krun_set_root: KrunSetRoot, pub krun_set_workdir: KrunSetWorkdir, pub krun_set_exec: KrunSetExec, - pub krun_set_port_map: KrunSetPortMap, pub krun_set_console_output: KrunSetConsoleOutput, pub krun_start_enter: KrunStartEnter, pub krun_disable_implicit_vsock: KrunDisableImplicitVsock, @@ -121,7 +119,6 @@ impl LibKrun { krun_set_root: load_symbol(library, b"krun_set_root\0", &libkrun_path)?, krun_set_workdir: load_symbol(library, b"krun_set_workdir\0", &libkrun_path)?, krun_set_exec: load_symbol(library, b"krun_set_exec\0", &libkrun_path)?, - krun_set_port_map: load_symbol(library, b"krun_set_port_map\0", &libkrun_path)?, krun_set_console_output: load_symbol( library, b"krun_set_console_output\0", diff --git a/crates/openshell-driver-vm/src/lib.rs b/crates/openshell-driver-vm/src/lib.rs index 1c424deebe..772db47b31 100644 --- a/crates/openshell-driver-vm/src/lib.rs +++ b/crates/openshell-driver-vm/src/lib.rs @@ -4,10 +4,9 @@ pub mod driver; mod embedded_runtime; mod ffi; +pub mod procguard; mod rootfs; mod runtime; -pub const GUEST_SSH_PORT: u16 = 2222; - pub use driver::{VmDriver, VmDriverConfig}; pub use runtime::{VM_RUNTIME_DIR_ENV, VmLaunchConfig, configured_runtime_dir, run_vm}; diff --git a/crates/openshell-driver-vm/src/main.rs b/crates/openshell-driver-vm/src/main.rs index 3a7976273d..5a675e78a8 100644 --- a/crates/openshell-driver-vm/src/main.rs +++ b/crates/openshell-driver-vm/src/main.rs @@ -6,7 +6,8 @@ use miette::{IntoDiagnostic, Result}; use openshell_core::VERSION; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; use openshell_driver_vm::{ - VM_RUNTIME_DIR_ENV, VmDriver, VmDriverConfig, VmLaunchConfig, configured_runtime_dir, run_vm, + VM_RUNTIME_DIR_ENV, VmDriver, VmDriverConfig, VmLaunchConfig, configured_runtime_dir, + procguard, run_vm, }; use std::net::SocketAddr; use std::path::PathBuf; @@ -34,9 +35,6 @@ struct Args { #[arg(long, hide = true)] vm_env: Vec, - #[arg(long, hide = true)] - vm_port: Vec, - #[arg(long, hide = true)] vm_console_output: Option, @@ -101,6 +99,14 @@ struct Args { async fn main() -> Result<()> { let args = Args::parse(); if args.internal_run_vm { + // We intentionally defer procguard arming until `run_vm()` so + // that the only arm is the one that knows how to clean up + // gvproxy. Racing two watchers against the same parent-death + // event causes the bare arm's `exit(1)` to win, skipping the + // gvproxy cleanup and leaking the helper. The risk window + // before `run_vm` arms procguard is ~a few syscalls long + // (`build_vm_launch_config`, `configured_runtime_dir`), which + // is negligible next to the parent gRPC server's uptime. maybe_reexec_internal_vm_with_runtime_env()?; let config = build_vm_launch_config(&args).map_err(|err| miette::miette!("{err}"))?; run_vm(&config).map_err(|err| miette::miette!("{err}"))?; @@ -113,6 +119,18 @@ async fn main() -> Result<()> { ) .init(); + // Arm procguard so that if the gateway is killed (SIGKILL or crash) + // we also die. Without this the driver is reparented to init and + // keeps its per-sandbox VM launchers alive forever. Launchers have + // their own procguards (armed in `run_vm`) which cascade cleanup of + // gvproxy and the libkrun worker the moment this driver exits. + if let Err(err) = procguard::die_with_parent() { + tracing::warn!( + error = %err, + "procguard arm failed; gateway crashes may orphan this driver" + ); + } + let driver = VmDriver::new(VmDriverConfig { openshell_endpoint: args .openshell_endpoint @@ -183,7 +201,6 @@ fn build_vm_launch_config(args: &Args) -> std::result::Result std::result::Result Result<()> { + use std::os::unix::process::CommandExt as _; + const REEXEC_ENV: &str = "__OPENSHELL_DRIVER_VM_REEXEC"; if std::env::var_os(REEXEC_ENV).is_some() { @@ -213,14 +232,23 @@ fn maybe_reexec_internal_vm_with_runtime_env() -> Result<()> { .map_err(|err| miette::miette!("join DYLD_LIBRARY_PATH: {err}"))?; let exe = std::env::current_exe().into_diagnostic()?; let args: Vec = std::env::args().skip(1).collect(); - let status = std::process::Command::new(exe) + + // Use execvp() so the current process is *replaced* by the re-exec'd + // binary — no wrapper process sits between the compute driver and + // the actually-running VM launcher. That avoids two problems: + // 1. An extra process level that survives SIGKILL of the driver + // (the wrapper was reparenting the re-exec'd child to init). + // 2. Signal forwarding: with a wrapper, a SIGTERM to the wrapper + // doesn't reach the child unless we hand-roll forwarding. + // After exec, the child inherits our PID and our procguard arming. + let err = std::process::Command::new(exe) .args(&args) .env("DYLD_LIBRARY_PATH", &joined) .env(VM_RUNTIME_DIR_ENV, runtime_dir) .env(REEXEC_ENV, "1") - .status() - .into_diagnostic()?; - std::process::exit(status.code().unwrap_or(1)); + .exec(); + // `exec()` only returns on failure. + Err(miette::miette!("failed to re-exec with runtime env: {err}")) } #[cfg(not(target_os = "macos"))] diff --git a/crates/openshell-driver-vm/src/procguard.rs b/crates/openshell-driver-vm/src/procguard.rs new file mode 100644 index 0000000000..1d91880f74 --- /dev/null +++ b/crates/openshell-driver-vm/src/procguard.rs @@ -0,0 +1,196 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Cross-platform "die when my parent dies" primitive. +//! +//! The VM driver spawns a chain of subprocesses (compute driver → `--internal-run-vm` +//! launcher → gvproxy + libkrun fork). If any link in that chain is killed +//! with SIGKILL — or simply crashes — the children are reparented to init +//! and survive indefinitely, leaking libkrun workers and gvproxy +//! instances. +//! +//! This module exposes two functions: +//! * [`die_with_parent`] — configure the kernel (Linux) or a helper +//! thread (BSDs, incl. macOS) to SIGKILL the current process when its +//! parent dies. Call it from `main` in every subprocess we spawn +//! along the chain. Idempotent-ish (each call is a full setup — see +//! the runtime.rs comment at the single call site). +//! * [`die_with_parent_cleanup`] — same as above, but on the BSD path a +//! best-effort cleanup callback runs *before* this process exits. +//! This matters when we own a non-Rust child (e.g. gvproxy) that +//! cannot arm its own procguard; the callback lets us SIGTERM it +//! first. +//! +//! The Linux path uses `nix::sys::prctl::set_pdeathsig(SIGKILL)`, and +//! the BSD path uses `smol-rs/polling` with its `kqueue::Process` + +//! `ProcessOps::Exit` filter. Both are well-tested library surfaces; +//! we keep only the glue code and the pre-arming parent-liveness +//! re-check. + +/// Arrange for the current process to receive SIGKILL if its parent dies. +/// +/// On Linux this sets `PR_SET_PDEATHSIG` to SIGKILL (via +/// `nix::sys::prctl`). The kernel delivers SIGKILL the moment +/// `getppid()` changes away from the original parent. +/// +/// On the BSD family (macOS, FreeBSD, etc.) this spawns a detached +/// helper thread that uses `kqueue` with `EVFILT_PROC | NOTE_EXIT` on +/// the parent PID. When the parent exits the thread calls `exit(1)`, +/// which is sufficient for our use case — we are not a critical daemon +/// that needs to drain state; we are a VM launcher / gRPC driver whose +/// entire job is tied to the parent's lifetime. +pub fn die_with_parent() -> Result<(), String> { + die_with_parent_cleanup(|| ()) +} + +/// Like [`die_with_parent`], but run `cleanup` (best-effort, +/// async-signal-unsafe — it runs on the helper thread) immediately +/// before terminating the process. Use this when we own children that +/// cannot arm their own procguard; the cleanup hook is the only chance +/// we get to send them SIGTERM after the kernel reparents us. +/// +/// On Linux the cleanup is a no-op: `PR_SET_PDEATHSIG` delivers SIGKILL +/// directly to us, there is no Rust-controlled moment between "parent +/// died" and "we die" in which we could run a callback. +pub fn die_with_parent_cleanup(cleanup: F) -> Result<(), String> +where + F: FnOnce() + Send + 'static, +{ + #[cfg(target_os = "linux")] + { + // Linux has no opportunity for a cleanup hook — the kernel + // delivers SIGKILL directly. Callers that need pre-exit cleanup + // must combine this with a `pre_exec` PR_SET_PDEATHSIG on their + // children (so the kernel cascades) or rely on process-group + // killpg from a signal handler in the parent. + let _ = cleanup; // intentionally dropped + install_linux_pdeathsig() + } + + #[cfg(any( + target_os = "macos", + target_os = "ios", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd", + target_os = "dragonfly", + ))] + { + install_bsd_kqueue_watcher(cleanup) + } + + #[cfg(not(any( + target_os = "linux", + target_os = "macos", + target_os = "ios", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd", + target_os = "dragonfly", + )))] + { + let _ = cleanup; + Ok(()) + } +} + +#[cfg(target_os = "linux")] +fn install_linux_pdeathsig() -> Result<(), String> { + use nix::sys::signal::Signal; + use nix::unistd::getppid; + + // Race: if the parent already died between fork/exec and this call, + // `getppid()` now returns 1 and PR_SET_PDEATHSIG will never fire. + // Read the current parent first so we can detect that case and exit. + let original_ppid = getppid(); + if original_ppid == nix::unistd::Pid::from_raw(1) { + return Err("process was already orphaned before procguard armed".to_string()); + } + + nix::sys::prctl::set_pdeathsig(Signal::SIGKILL) + .map_err(|err| format!("prctl(PR_SET_PDEATHSIG) failed: {err}"))?; + + // Re-check after arming: the parent may have died between getppid() + // and prctl(). If so, PR_SET_PDEATHSIG missed its window. + if getppid() != original_ppid { + return Err("parent exited before procguard could arm".to_string()); + } + + Ok(()) +} + +#[cfg(any( + target_os = "macos", + target_os = "ios", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd", + target_os = "dragonfly", +))] +fn install_bsd_kqueue_watcher(cleanup: F) -> Result<(), String> +where + F: FnOnce() + Send + 'static, +{ + use nix::unistd::getppid; + use polling::os::kqueue::{PollerKqueueExt, Process, ProcessOps}; + use polling::{Events, PollMode, Poller}; + + let parent_pid = getppid(); + if parent_pid == nix::unistd::Pid::from_raw(1) { + return Err("process was already orphaned before procguard armed".to_string()); + } + let parent_pid_nz = std::num::NonZeroI32::new(parent_pid.as_raw()) + .ok_or_else(|| "getppid returned 0 unexpectedly".to_string())?; + + // Build the poller on the caller's thread so any setup error + // surfaces synchronously. `EVFILT_PROC | NOTE_EXIT` is a one-shot + // filter, so `PollMode::Oneshot` matches the kernel semantics. + // + // SAFETY: `Process::from_pid` requires the PID to "be tied to an + // actual child process". Our parent is alive at this point — we + // re-check `getppid()` immediately after registration to close the + // race where the parent dies between the read above and the + // `add_filter` call. The BSD kqueue implementation accepts any + // live PID, not just our own children; the "child" wording in the + // polling docs is carried over from historical terminology in the + // kqueue(2) manpage. The kernel guarantees NOTE_EXIT fires if the + // PID is valid at registration. + let poller = Poller::new().map_err(|err| format!("polling: Poller::new failed: {err}"))?; + let key = 1; + #[allow(unsafe_code)] + // SAFETY requirement is documented on the enclosing function: the + // PID was just read from `getppid()` and re-checked below, so it + // points at a live process. `Process::from_pid` is an + // entry-in-the-kernel-table registration — the kernel validates + // the PID when the filter is added. + let filter = unsafe { Process::from_pid(parent_pid_nz, ProcessOps::Exit) }; + poller + .add_filter(filter, key, PollMode::Oneshot) + .map_err(|err| format!("polling: add_filter(NOTE_EXIT, {parent_pid_nz}) failed: {err}"))?; + + // Between getppid() and the registered filter the parent may + // already have died. Detect that and abort so the caller can bail. + if getppid() != parent_pid { + return Err("parent exited before procguard could arm".to_string()); + } + + // Hand off to a dedicated OS thread. Block in `poller.wait()` + // until the single NOTE_EXIT event fires, run the cleanup, then + // exit. We prefer `exit(1)` over `kill(getpid, SIGKILL)` so the + // callback gets to complete — SIGKILL would race it. Our children + // have their own procguards armed and will notice `getppid() == + // 1` shortly after, so we do not need Linux-semantics exactness. + std::thread::Builder::new() + .name("procguard".to_string()) + .spawn(move || { + let mut events = Events::new(); + // Block indefinitely; the filter is Oneshot so we expect + // exactly one event (parent's NOTE_EXIT) or a spurious + // wakeup we treat the same way. + let _ = poller.wait(&mut events, None); + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(cleanup)); + std::process::exit(1); + }) + .map(|_| ()) + .map_err(|e| format!("failed to spawn procguard thread: {e}")) +} diff --git a/crates/openshell-driver-vm/src/runtime.rs b/crates/openshell-driver-vm/src/runtime.rs index 9888feb182..e20c7d4e51 100644 --- a/crates/openshell-driver-vm/src/runtime.rs +++ b/crates/openshell-driver-vm/src/runtime.rs @@ -4,20 +4,25 @@ #![allow(unsafe_code)] use std::ffi::CString; -use std::io::{Read, Write}; -use std::os::unix::net::UnixStream; use std::path::{Path, PathBuf}; use std::process::{Child as StdChild, Command as StdCommand, Stdio}; use std::ptr; use std::sync::atomic::{AtomicI32, Ordering}; use std::time::{Duration, Instant}; -use crate::{GUEST_SSH_PORT, embedded_runtime, ffi}; +use crate::{embedded_runtime, ffi, procguard}; pub const VM_RUNTIME_DIR_ENV: &str = "OPENSHELL_VM_RUNTIME_DIR"; +/// PID of the forked libkrun worker (the VM's PID 1). Zero when not running. +/// Used by the SIGTERM/SIGINT handler to forward signals to the VM. static CHILD_PID: AtomicI32 = AtomicI32::new(0); +/// PID of the gvproxy helper process. Zero when not running. Used by the +/// SIGTERM/SIGINT handler to make sure gvproxy doesn't survive the +/// launcher on macOS (where we can't use `PR_SET_PDEATHSIG`). +static GVPROXY_PID: AtomicI32 = AtomicI32::new(0); + pub struct VmLaunchConfig { pub rootfs: PathBuf, pub vcpus: u8, @@ -26,23 +31,10 @@ pub struct VmLaunchConfig { pub args: Vec, pub env: Vec, pub workdir: String, - pub port_map: Vec, pub log_level: u32, pub console_output: PathBuf, } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -struct PortMapping { - host_port: u16, - guest_port: u16, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -struct GvproxyPortPlan { - ssh_port: u16, - forwarded_ports: Vec, -} - pub fn run_vm(config: &VmLaunchConfig) -> Result<(), String> { if !config.rootfs.is_dir() { return Err(format!( @@ -51,6 +43,47 @@ pub fn run_vm(config: &VmLaunchConfig) -> Result<(), String> { )); } + // Arm procguard first, BEFORE we spawn gvproxy or fork libkrun, so + // that the launcher can't be orphaned during setup. The cleanup + // callback reads the GVPROXY_PID atomic (initially 0 — no-op) and + // the CHILD_PID atomic (the libkrun fork), so it stays correct as + // those slots get populated later in this function. Only ONE arm + // per process: racing two watchers for the same NOTE_EXIT event + // would cause whichever wins to skip the cleanup. + if let Err(err) = procguard::die_with_parent_cleanup(|| { + // Cleanup order: SIGTERM gvproxy and the libkrun fork first so + // they can drain cleanly, then SIGKILL after a brief grace + // window. We can't rely on Rust destructors here; when + // procguard's watcher thread returns we call `std::process::exit` + // and the process tears down. Only async-signal-safe calls here: + // atomic loads and `kill(2)` are both on the POSIX list. + let gv_pid = GVPROXY_PID.load(Ordering::Relaxed); + let child_pid = CHILD_PID.load(Ordering::Relaxed); + if gv_pid > 0 { + unsafe { + libc::kill(gv_pid, libc::SIGTERM); + } + } + if child_pid > 0 { + unsafe { + libc::kill(child_pid, libc::SIGTERM); + } + } + std::thread::sleep(Duration::from_millis(200)); + if gv_pid > 0 { + unsafe { + libc::kill(gv_pid, libc::SIGKILL); + } + } + if child_pid > 0 { + unsafe { + libc::kill(child_pid, libc::SIGKILL); + } + } + }) { + return Err(format!("procguard arm failed: {err}")); + } + #[cfg(target_os = "linux")] check_kvm_access()?; @@ -64,10 +97,40 @@ pub fn run_vm(config: &VmLaunchConfig) -> Result<(), String> { vm.set_root(&config.rootfs)?; vm.set_workdir(&config.workdir)?; - let mut forwarded_port_map = config.port_map.clone(); - let mut gvproxy_guard = None; - let mut gvproxy_api_sock = None; - if !config.port_map.is_empty() { + // Run gvproxy strictly as the guest's virtual NIC / DHCP / router. + // + // After the supervisor-initiated relay migration (#867), the driver + // no longer forwards any host-side ports into the guest — all ingress + // traffic for SSH and exec rides the outbound `ConnectSupervisor` + // gRPC stream the guest opens to the gateway. What gvproxy still + // provides here is the TCP/IP *plane* the guest kernel needs: + // + // * a virtio-net backend attached to libkrun via a Unix + // SOCK_STREAM (Linux) or SOCK_DGRAM (macOS vfkit), which + // surfaces as `eth0` inside the guest; + // * the DHCP server + default router the guest's udhcpc client + // talks to on boot (IPs 192.168.127.1 / .2, defaults for + // gvisor-tap-vsock); + // * the host-facing gateway identity the guest uses for callbacks: + // the init script seeds `/etc/hosts` with + // `host.openshell.internal` pointing at 192.168.127.1 while + // leaving gvproxy's legacy `host.containers.internal` / + // `host.docker.internal` DNS answers intact, which is how the guest's + // `rewrite_openshell_endpoint_if_needed` probe reaches the host + // gateway when the bare loopback address doesn't resolve from + // inside the VM. + // + // That network plane is also what the sandbox supervisor's + // per-sandbox netns (veth pair + iptables, see + // `openshell-sandbox/src/sandbox/linux/netns.rs`) branches off of; + // libkrun's built-in TSI socket impersonation would not satisfy + // those kernel-level primitives. + // + // The `-listen` API socket and `-ssh-port` forwarder are both + // deliberately omitted: nothing in the driver enqueues port + // forwards on the API any more, and the host-side SSH listener is + // dead plumbing. + let gvproxy_guard = { let gvproxy_binary = runtime_dir.join("gvproxy"); if !gvproxy_binary.is_file() { return Err(format!( @@ -76,13 +139,9 @@ pub fn run_vm(config: &VmLaunchConfig) -> Result<(), String> { )); } - kill_stale_gvproxy_by_port_map(&config.port_map); - let sock_base = gvproxy_socket_base(&config.rootfs)?; let net_sock = sock_base.with_extension("v"); - let api_sock = sock_base.with_extension("a"); let _ = std::fs::remove_file(&net_sock); - let _ = std::fs::remove_file(&api_sock); let _ = std::fs::remove_file(sock_base.with_extension("v-krun.sock")); let run_dir = config.rootfs.parent().unwrap_or(&config.rootfs); @@ -90,9 +149,6 @@ pub fn run_vm(config: &VmLaunchConfig) -> Result<(), String> { let gvproxy_log_file = std::fs::File::create(&gvproxy_log) .map_err(|e| format!("create gvproxy log {}: {e}", gvproxy_log.display()))?; - let gvproxy_ports = plan_gvproxy_ports(&config.port_map)?; - forwarded_port_map = gvproxy_ports.forwarded_ports; - #[cfg(target_os = "linux")] let (gvproxy_net_flag, gvproxy_net_url) = ("-listen-qemu", format!("unix://{}", net_sock.display())); @@ -102,18 +158,51 @@ pub fn run_vm(config: &VmLaunchConfig) -> Result<(), String> { format!("unixgram://{}", net_sock.display()), ); - let child = StdCommand::new(&gvproxy_binary) + // `-ssh-port -1` tells gvproxy to skip its default SSH forward + // (127.0.0.1:2222 → guest:22). We don't use it — all gateway + // ingress rides the supervisor-initiated relay — and leaving + // the default on would bind a host-side TCP listener per + // sandbox, racing concurrent sandboxes for port 2222 and + // surfacing a misleading "sshd is reachable" endpoint. See + // https://github.com/containers/gvisor-tap-vsock `cmd/gvproxy/main.go` + // (`getForwardsMap` returns an empty map when `sshPort == -1`). + let mut gvproxy_cmd = StdCommand::new(&gvproxy_binary); + gvproxy_cmd .arg(gvproxy_net_flag) .arg(&gvproxy_net_url) - .arg("-listen") - .arg(format!("unix://{}", api_sock.display())) .arg("-ssh-port") - .arg(gvproxy_ports.ssh_port.to_string()) + .arg("-1") .stdin(Stdio::null()) .stdout(Stdio::null()) - .stderr(gvproxy_log_file) + .stderr(gvproxy_log_file); + + // On Linux the kernel will SIGKILL gvproxy the moment this + // launcher dies (or is SIGKILLed). `pre_exec` runs in the child + // between fork and execve, so the PR_SET_PDEATHSIG flag is + // inherited across execve and applies to gvproxy proper. On + // macOS/BSDs there is no equivalent; we fall back to killing + // gvproxy explicitly from the launcher's procguard cleanup + // callback (see `run_vm` above) and SIGTERM handler + // (see `install_signal_forwarding` below). + #[cfg(target_os = "linux")] + { + use nix::sys::signal::Signal; + use std::os::unix::process::CommandExt as _; + unsafe { + gvproxy_cmd.pre_exec(|| { + nix::sys::prctl::set_pdeathsig(Signal::SIGKILL) + .map_err(|err| std::io::Error::other(format!("pdeathsig: {err}"))) + }); + } + } + + let child = gvproxy_cmd .spawn() .map_err(|e| format!("failed to start gvproxy {}: {e}", gvproxy_binary.display()))?; + // The procguard cleanup reads GVPROXY_PID atomically. Storing it + // here makes the callback able to SIGTERM gvproxy if the driver + // dies from this moment onward. + GVPROXY_PID.store(child.id() as i32, Ordering::Relaxed); wait_for_path(&net_sock, Duration::from_secs(5), "gvproxy data socket")?; @@ -142,13 +231,9 @@ pub fn run_vm(config: &VmLaunchConfig) -> Result<(), String> { vm.add_net_unixgram(&net_sock, &mac, COMPAT_NET_FEATURES, NET_FLAG_VFKIT)?; } - gvproxy_guard = Some(GvproxyGuard::new(child)); - gvproxy_api_sock = Some(api_sock); - } + Some(GvproxyGuard::new(child)) + }; - if !config.port_map.is_empty() && gvproxy_api_sock.is_none() { - vm.set_port_map(&config.port_map)?; - } vm.set_console_output(&config.console_output)?; let env = if config.env.is_empty() { @@ -166,6 +251,20 @@ pub fn run_vm(config: &VmLaunchConfig) -> Result<(), String> { match pid { -1 => Err(format!("fork failed: {}", std::io::Error::last_os_error())), 0 => { + // We are the libkrun worker (the VM's PID 1 inside the guest + // kernel, but a normal host process until krun_start_enter + // fires). Arm procguard so this fork is SIGKILLed if the + // parent launcher dies abruptly. On Linux this uses + // `PR_SET_PDEATHSIG`; on macOS this spawns a kqueue + // NOTE_EXIT watcher thread. Either way it closes the same + // leak gvproxy does above. + // + // We also SIGKILL ourselves if arming fails — there's no + // safe way to continue if we can't guarantee cleanup. + if let Err(err) = procguard::die_with_parent() { + eprintln!("libkrun worker: procguard arm failed: {err}"); + std::process::exit(1); + } let ret = vm.start_enter(); eprintln!("krun_start_enter failed: {ret}"); std::process::exit(1); @@ -173,24 +272,10 @@ pub fn run_vm(config: &VmLaunchConfig) -> Result<(), String> { _ => { install_signal_forwarding(pid); - let port_forward_result = if let Some(api_sock) = gvproxy_api_sock.as_ref() { - expose_port_map(api_sock, &forwarded_port_map) - } else { - Ok(()) - }; - - if let Err(err) = port_forward_result { - unsafe { - libc::kill(pid, libc::SIGTERM); - } - let _ = wait_for_child(pid); - cleanup_gvproxy(gvproxy_guard); - return Err(err); - } - let status = wait_for_child(pid)?; CHILD_PID.store(0, Ordering::Relaxed); cleanup_gvproxy(gvproxy_guard); + GVPROXY_PID.store(0, Ordering::Relaxed); if libc::WIFEXITED(status) { match libc::WEXITSTATUS(status) { @@ -399,15 +484,6 @@ impl VmContext { ) } - fn set_port_map(&self, port_map: &[String]) -> Result<(), String> { - let port_strs: Vec<&str> = port_map.iter().map(String::as_str).collect(); - let (_owners, ptrs) = c_string_array(&port_strs)?; - check( - unsafe { (self.krun.krun_set_port_map)(self.ctx_id, ptrs.as_ptr()) }, - "krun_set_port_map", - ) - } - fn set_console_output(&self, path: &Path) -> Result<(), String> { let console_c = path_to_cstring(path)?; check( @@ -476,126 +552,6 @@ impl Drop for GvproxyGuard { } } -fn expose_port_map(api_sock: &Path, port_map: &[String]) -> Result<(), String> { - wait_for_path(api_sock, Duration::from_secs(2), "gvproxy API socket")?; - let guest_ip = "192.168.127.2"; - - for pm in port_map { - let mapping = parse_port_mapping(pm)?; - - let expose_body = format!( - r#"{{"local":":{}","remote":"{guest_ip}:{}","protocol":"tcp"}}"#, - mapping.host_port, mapping.guest_port - ); - - let deadline = Instant::now() + Duration::from_secs(10); - let mut retry_interval = Duration::from_millis(100); - loop { - match gvproxy_expose(api_sock, &expose_body) { - Ok(()) => break, - Err(err) if Instant::now() < deadline => { - std::thread::sleep(retry_interval); - retry_interval = (retry_interval * 2).min(Duration::from_secs(1)); - if retry_interval == Duration::from_secs(1) { - eprintln!("retrying gvproxy port expose {pm}: {err}"); - } - } - Err(err) => { - return Err(format!( - "failed to forward port {} via gvproxy: {err}", - mapping.host_port - )); - } - } - } - } - - Ok(()) -} - -fn gvproxy_expose(api_sock: &Path, body: &str) -> Result<(), String> { - let mut stream = - UnixStream::connect(api_sock).map_err(|e| format!("connect to gvproxy API socket: {e}"))?; - - let request = format!( - "POST /services/forwarder/expose HTTP/1.1\r\n\ - Host: localhost\r\n\ - Content-Type: application/json\r\n\ - Content-Length: {}\r\n\ - Connection: close\r\n\ - \r\n\ - {}", - body.len(), - body, - ); - - stream - .write_all(request.as_bytes()) - .map_err(|e| format!("write to gvproxy API: {e}"))?; - - let mut buf = [0u8; 1024]; - let n = stream - .read(&mut buf) - .map_err(|e| format!("read from gvproxy API: {e}"))?; - let response = String::from_utf8_lossy(&buf[..n]); - let status = response - .lines() - .next() - .and_then(|line| line.split_whitespace().nth(1)) - .unwrap_or("0"); - - match status { - "200" | "204" => Ok(()), - _ => Err(format!( - "gvproxy API: {}", - response.lines().next().unwrap_or("") - )), - } -} - -fn plan_gvproxy_ports(port_map: &[String]) -> Result { - let mut ssh_port = None; - let mut forwarded_ports = Vec::with_capacity(port_map.len()); - - for pm in port_map { - let mapping = parse_port_mapping(pm)?; - if ssh_port.is_none() && mapping.guest_port == GUEST_SSH_PORT && mapping.host_port >= 1024 { - ssh_port = Some(mapping.host_port); - continue; - } - forwarded_ports.push(pm.clone()); - } - - Ok(GvproxyPortPlan { - ssh_port: match ssh_port { - Some(port) => port, - None => pick_gvproxy_ssh_port()?, - }, - forwarded_ports, - }) -} - -fn parse_port_mapping(pm: &str) -> Result { - let parts: Vec<&str> = pm.split(':').collect(); - let (host, guest) = match parts.as_slice() { - [host, guest] => (*host, *guest), - [port] => (*port, *port), - _ => return Err(format!("invalid port mapping '{pm}'")), - }; - - let host_port = host - .parse::() - .map_err(|_| format!("invalid port mapping '{pm}'"))?; - let guest_port = guest - .parse::() - .map_err(|_| format!("invalid port mapping '{pm}'"))?; - - Ok(PortMapping { - host_port, - guest_port, - }) -} - fn wait_for_path(path: &Path, timeout: Duration, label: &str) -> Result<(), String> { let deadline = Instant::now() + timeout; let mut interval = Duration::from_millis(5); @@ -674,92 +630,6 @@ fn gvproxy_socket_base(rootfs: &Path) -> Result { Ok(secure_socket_base("osd-gv")?.join(hash_path_id(rootfs))) } -fn pick_gvproxy_ssh_port() -> Result { - let listener = std::net::TcpListener::bind(("127.0.0.1", 0)) - .map_err(|e| format!("allocate gvproxy ssh port on localhost: {e}"))?; - let port = listener - .local_addr() - .map_err(|e| format!("read gvproxy ssh port: {e}"))? - .port(); - drop(listener); - Ok(port) -} - -fn kill_stale_gvproxy_by_port_map(port_map: &[String]) { - for pm in port_map { - if let Some(host_port) = pm - .split(':') - .next() - .and_then(|port| port.parse::().ok()) - { - kill_stale_gvproxy_by_port(host_port); - } - } -} - -fn kill_stale_gvproxy_by_port(port: u16) { - let output = StdCommand::new("lsof") - .args(["-ti", &format!(":{port}")]) - .output(); - - let pids = match output { - Ok(output) if output.status.success() => { - String::from_utf8_lossy(&output.stdout).to_string() - } - _ => return, - }; - - for line in pids.lines() { - if let Ok(pid) = line.trim().parse::() - && is_process_named(pid as libc::pid_t, "gvproxy") - { - kill_gvproxy_pid(pid); - } - } -} - -fn kill_gvproxy_pid(pid: u32) { - let pid = pid as libc::pid_t; - if unsafe { libc::kill(pid, 0) } != 0 { - return; - } - if !is_process_named(pid, "gvproxy") { - return; - } - unsafe { - libc::kill(pid, libc::SIGTERM); - } - std::thread::sleep(Duration::from_millis(200)); -} - -#[cfg(target_os = "macos")] -fn is_process_named(pid: libc::pid_t, expected: &str) -> bool { - StdCommand::new("ps") - .args(["-p", &pid.to_string(), "-o", "comm="]) - .output() - .ok() - .and_then(|output| { - if output.status.success() { - String::from_utf8(output.stdout).ok() - } else { - None - } - }) - .is_some_and(|name| name.trim().contains(expected)) -} - -#[cfg(target_os = "linux")] -fn is_process_named(pid: libc::pid_t, expected: &str) -> bool { - std::fs::read_to_string(format!("/proc/{pid}/comm")) - .map(|name| name.trim().contains(expected)) - .unwrap_or(false) -} - -#[cfg(not(any(target_os = "macos", target_os = "linux")))] -fn is_process_named(_pid: libc::pid_t, _expected: &str) -> bool { - false -} - fn install_signal_forwarding(pid: i32) { unsafe { libc::signal( @@ -774,11 +644,28 @@ fn install_signal_forwarding(pid: i32) { CHILD_PID.store(pid, Ordering::Relaxed); } +/// Async-signal-safe handler that forwards SIGTERM to every process we +/// own: the libkrun VM worker and the gvproxy helper. We cannot rely on +/// Rust destructors (`GvproxyGuard::drop`, `ManagedDriverProcess::drop`) +/// running on signal-driven exit, so we explicitly deliver the signal +/// here. The `wait_for_child` loop reaps libkrun and `cleanup_gvproxy` +/// reaps gvproxy before `run_vm` returns. +/// +/// Only async-signal-safe libc calls are used — `kill(2)` is listed in +/// POSIX.1-2017 as async-signal-safe, atomic loads are lock-free on the +/// platforms we target. extern "C" fn forward_signal(_sig: libc::c_int) { - let pid = CHILD_PID.load(Ordering::Relaxed); - if pid > 0 { + let vm_pid = CHILD_PID.load(Ordering::Relaxed); + if vm_pid > 0 { unsafe { - libc::kill(pid, libc::SIGTERM); + libc::kill(vm_pid, libc::SIGTERM); + } + } + let gv_pid = GVPROXY_PID.load(Ordering::Relaxed); + if gv_pid > 0 { + // gvproxy handles SIGTERM cleanly; no need for SIGKILL. + unsafe { + libc::kill(gv_pid, libc::SIGTERM); } } } @@ -840,38 +727,3 @@ fn check_kvm_access() -> Result<(), String> { format!("cannot open /dev/kvm: {e}\nKVM access is required to run microVMs on Linux.") }) } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn plan_gvproxy_ports_reuses_sandbox_ssh_mapping() { - let plan = plan_gvproxy_ports(&["64739:2222".to_string()]).expect("plan should succeed"); - - assert_eq!(plan.ssh_port, 64739); - assert!(plan.forwarded_ports.is_empty()); - } - - #[test] - fn plan_gvproxy_ports_keeps_non_ssh_mappings_for_forwarder() { - let plan = plan_gvproxy_ports(&["64739:8080".to_string()]).expect("plan should succeed"); - - assert_ne!(plan.ssh_port, 64739); - assert_eq!(plan.forwarded_ports, vec!["64739:8080".to_string()]); - } - - #[test] - fn plan_gvproxy_ports_ignores_privileged_host_ports_for_direct_ssh() { - let plan = plan_gvproxy_ports(&["22:2222".to_string()]).expect("plan should succeed"); - - assert_ne!(plan.ssh_port, 22); - assert_eq!(plan.forwarded_ports, vec!["22:2222".to_string()]); - } - - #[test] - fn parse_port_mapping_rejects_invalid_entries() { - let err = parse_port_mapping("bad:mapping").expect_err("invalid mapping should fail"); - assert!(err.contains("invalid port mapping")); - } -} diff --git a/crates/openshell-driver-vm/start.sh b/crates/openshell-driver-vm/start.sh index 155136c785..b5aebbefd8 100755 --- a/crates/openshell-driver-vm/start.sh +++ b/crates/openshell-driver-vm/start.sh @@ -5,12 +5,26 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +CLI_BIN="${ROOT}/scripts/bin/openshell" COMPRESSED_DIR="${ROOT}/target/vm-runtime-compressed" -STATE_DIR_DEFAULT="${ROOT}/target/openshell-vm-driver-dev" +SERVER_PORT="${OPENSHELL_SERVER_PORT:-8080}" +# Keep the driver socket path under AF_UNIX SUN_LEN on macOS. +STATE_DIR_ROOT="${OPENSHELL_VM_DRIVER_STATE_ROOT:-/tmp}" +STATE_LABEL_RAW="${OPENSHELL_VM_INSTANCE:-port-${SERVER_PORT}}" +STATE_LABEL="$(printf '%s' "${STATE_LABEL_RAW}" | tr -cs '[:alnum:]._-' '-')" +if [ -z "${STATE_LABEL}" ]; then + STATE_LABEL="port-${SERVER_PORT}" +fi +STATE_DIR_DEFAULT="${STATE_DIR_ROOT}/openshell-vm-driver-dev-${USER:-user}-${STATE_LABEL}" STATE_DIR="${OPENSHELL_VM_DRIVER_STATE_DIR:-${STATE_DIR_DEFAULT}}" DB_PATH_DEFAULT="${STATE_DIR}/openshell.db" -SERVER_PORT="${OPENSHELL_SERVER_PORT:-8080}" VM_HOST_GATEWAY_DEFAULT="${OPENSHELL_VM_HOST_GATEWAY:-host.containers.internal}" +LOCAL_GATEWAY_ENDPOINT_DEFAULT="http://127.0.0.1:${SERVER_PORT}" +LOCAL_GATEWAY_ENDPOINT="${OPENSHELL_VM_LOCAL_GATEWAY_ENDPOINT:-${LOCAL_GATEWAY_ENDPOINT_DEFAULT}}" +GATEWAY_NAME_DEFAULT="vm-driver-${STATE_LABEL}" +GATEWAY_NAME="${OPENSHELL_VM_GATEWAY_NAME:-${GATEWAY_NAME_DEFAULT}}" +DRIVER_DIR_DEFAULT="${ROOT}/target/debug" +DRIVER_DIR="${OPENSHELL_DRIVER_DIR:-${DRIVER_DIR_DEFAULT}}" export OPENSHELL_VM_RUNTIME_COMPRESSED_DIR="${OPENSHELL_VM_RUNTIME_COMPRESSED_DIR:-${COMPRESSED_DIR}}" @@ -52,11 +66,19 @@ fi export OPENSHELL_DISABLE_TLS="$(normalize_bool "${OPENSHELL_DISABLE_TLS:-true}")" export OPENSHELL_DB_URL="${OPENSHELL_DB_URL:-sqlite:${DB_PATH_DEFAULT}}" export OPENSHELL_DRIVERS="${OPENSHELL_DRIVERS:-vm}" +export OPENSHELL_DRIVER_DIR="${DRIVER_DIR}" export OPENSHELL_GRPC_ENDPOINT="${OPENSHELL_GRPC_ENDPOINT:-http://${VM_HOST_GATEWAY_DEFAULT}:${SERVER_PORT}}" export OPENSHELL_SSH_GATEWAY_HOST="${OPENSHELL_SSH_GATEWAY_HOST:-127.0.0.1}" export OPENSHELL_SSH_GATEWAY_PORT="${OPENSHELL_SSH_GATEWAY_PORT:-${SERVER_PORT}}" export OPENSHELL_SSH_HANDSHAKE_SECRET="${OPENSHELL_SSH_HANDSHAKE_SECRET:-dev-vm-driver-secret}" export OPENSHELL_VM_DRIVER_STATE_DIR="${STATE_DIR}" +echo "==> Gateway registration" +echo " Name: ${GATEWAY_NAME}" +echo " Endpoint: ${LOCAL_GATEWAY_ENDPOINT}" +echo " Register: ${CLI_BIN} gateway add --name ${GATEWAY_NAME} ${LOCAL_GATEWAY_ENDPOINT}" +echo " Select: ${CLI_BIN} gateway select ${GATEWAY_NAME}" +echo " Driver: ${OPENSHELL_DRIVER_DIR}/openshell-driver-vm" + echo "==> Starting OpenShell server with VM compute driver" exec "${ROOT}/target/debug/openshell-gateway" diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 95ffbfaa49..35c72f80c1 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -11,6 +11,7 @@ use crate::grpc::policy::{SANDBOX_SETTINGS_OBJECT_TYPE, sandbox_settings_id}; use crate::persistence::{ObjectId, ObjectName, ObjectRecord, ObjectType, Store}; use crate::sandbox_index::SandboxIndex; use crate::sandbox_watch::SandboxWatchBus; +use crate::supervisor_session::SupervisorSessionRegistry; use crate::tracing_bus::TracingLogBus; use futures::{Stream, StreamExt}; use openshell_core::proto::compute::v1::{ @@ -188,6 +189,7 @@ pub struct ComputeRuntime { sandbox_index: SandboxIndex, sandbox_watch_bus: SandboxWatchBus, tracing_log_bus: TracingLogBus, + supervisor_sessions: Arc, sync_lock: Arc>, } @@ -205,6 +207,7 @@ impl ComputeRuntime { sandbox_index: SandboxIndex, sandbox_watch_bus: SandboxWatchBus, tracing_log_bus: TracingLogBus, + supervisor_sessions: Arc, ) -> Result { let default_image = driver .get_capabilities(Request::new(GetCapabilitiesRequest {})) @@ -220,6 +223,7 @@ impl ComputeRuntime { sandbox_index, sandbox_watch_bus, tracing_log_bus, + supervisor_sessions, sync_lock: Arc::new(Mutex::new(())), }) } @@ -230,6 +234,7 @@ impl ComputeRuntime { sandbox_index: SandboxIndex, sandbox_watch_bus: SandboxWatchBus, tracing_log_bus: TracingLogBus, + supervisor_sessions: Arc, ) -> Result { let driver = KubernetesComputeDriver::new(config) .await @@ -242,6 +247,7 @@ impl ComputeRuntime { sandbox_index, sandbox_watch_bus, tracing_log_bus, + supervisor_sessions, ) .await } @@ -253,6 +259,7 @@ impl ComputeRuntime { sandbox_index: SandboxIndex, sandbox_watch_bus: SandboxWatchBus, tracing_log_bus: TracingLogBus, + supervisor_sessions: Arc, ) -> Result { let driver: SharedComputeDriver = Arc::new(RemoteComputeDriver::new(channel)); Self::from_driver( @@ -262,6 +269,7 @@ impl ComputeRuntime { sandbox_index, sandbox_watch_bus, tracing_log_bus, + supervisor_sessions, ) .await } @@ -563,7 +571,8 @@ impl ComputeRuntime { existing.as_ref().and_then(|sandbox| sandbox.spec.as_ref()), ); - let phase = derive_phase(incoming.status.as_ref()); + let session_connected = self.supervisor_sessions.has_session(&incoming.id); + let mut phase = derive_phase(incoming.status.as_ref()); let mut sandbox = existing.unwrap_or_else(|| Sandbox { id: incoming.id.clone(), name: incoming.name.clone(), @@ -574,6 +583,12 @@ impl ComputeRuntime { ..Default::default() }); + if session_connected && matches!(phase, SandboxPhase::Provisioning | SandboxPhase::Unknown) + { + ensure_supervisor_ready_status(&mut status, &sandbox.name); + phase = SandboxPhase::Ready; + } + let old_phase = SandboxPhase::try_from(sandbox.phase).unwrap_or(SandboxPhase::Unknown); if old_phase != phase { info!( @@ -622,6 +637,55 @@ impl ComputeRuntime { Ok(()) } + pub async fn supervisor_session_connected(&self, sandbox_id: &str) -> Result<(), String> { + self.set_supervisor_session_state(sandbox_id, true).await + } + + pub async fn supervisor_session_disconnected(&self, sandbox_id: &str) -> Result<(), String> { + self.set_supervisor_session_state(sandbox_id, false).await + } + + async fn set_supervisor_session_state( + &self, + sandbox_id: &str, + connected: bool, + ) -> Result<(), String> { + let _guard = self.sync_lock.lock().await; + let Some(record) = self + .store + .get(Sandbox::object_type(), sandbox_id) + .await + .map_err(|e| e.to_string())? + else { + return Ok(()); + }; + + let mut sandbox = decode_sandbox_record(&record)?; + let current_phase = SandboxPhase::try_from(sandbox.phase).unwrap_or(SandboxPhase::Unknown); + + if current_phase == SandboxPhase::Deleting || current_phase == SandboxPhase::Error { + return Ok(()); + } + + if connected { + ensure_supervisor_ready_status(&mut sandbox.status, &sandbox.name); + sandbox.phase = SandboxPhase::Ready as i32; + } else if current_phase == SandboxPhase::Ready { + ensure_supervisor_not_ready_status(&mut sandbox.status, &sandbox.name); + sandbox.phase = SandboxPhase::Provisioning as i32; + } else { + return Ok(()); + } + + self.sandbox_index.update_from_sandbox(&sandbox); + self.store + .put_message(&sandbox) + .await + .map_err(|e| e.to_string())?; + self.sandbox_watch_bus.notify(sandbox_id); + Ok(()) + } + async fn apply_deleted(&self, sandbox_id: &str) -> Result<(), String> { let _guard = self.sync_lock.lock().await; self.apply_deleted_locked(sandbox_id).await @@ -963,6 +1027,58 @@ fn public_status_from_driver(status: &DriverSandboxStatus) -> SandboxStatus { } } +fn ensure_supervisor_ready_status(status: &mut Option, sandbox_name: &str) { + upsert_ready_condition( + status, + sandbox_name, + SandboxCondition { + r#type: "Ready".to_string(), + status: "True".to_string(), + reason: "DependenciesReady".to_string(), + message: "Supervisor session connected".to_string(), + last_transition_time: String::new(), + }, + ); +} + +fn ensure_supervisor_not_ready_status(status: &mut Option, sandbox_name: &str) { + upsert_ready_condition( + status, + sandbox_name, + SandboxCondition { + r#type: "Ready".to_string(), + status: "False".to_string(), + reason: "DependenciesNotReady".to_string(), + message: "Supervisor session disconnected".to_string(), + last_transition_time: String::new(), + }, + ); +} + +fn upsert_ready_condition( + status: &mut Option, + sandbox_name: &str, + condition: SandboxCondition, +) { + let status = status.get_or_insert_with(|| SandboxStatus { + sandbox_name: sandbox_name.to_string(), + agent_pod: String::new(), + agent_fd: String::new(), + sandbox_fd: String::new(), + conditions: Vec::new(), + }); + + if let Some(existing) = status + .conditions + .iter_mut() + .find(|existing| existing.r#type == "Ready") + { + *existing = condition; + } else { + status.conditions.push(condition); + } +} + fn public_condition_from_driver(condition: &DriverCondition) -> SandboxCondition { SandboxCondition { r#type: condition.r#type.clone(), @@ -1044,6 +1160,7 @@ mod tests { GetSandboxResponse, StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateResponse, }; use std::sync::Arc; + use tokio::sync::{mpsc, oneshot}; #[derive(Debug, Default)] struct TestDriver { @@ -1159,10 +1276,22 @@ mod tests { sandbox_index: SandboxIndex::new(), sandbox_watch_bus: SandboxWatchBus::new(), tracing_log_bus: TracingLogBus::new(), + supervisor_sessions: Arc::new(SupervisorSessionRegistry::new()), sync_lock: Arc::new(Mutex::new(())), } } + fn register_test_supervisor_session(runtime: &ComputeRuntime, sandbox_id: &str) { + let (tx, _rx) = mpsc::channel(1); + let (shutdown_tx, _shutdown_rx) = oneshot::channel(); + runtime.supervisor_sessions.register( + sandbox_id.to_string(), + "session-1".to_string(), + tx, + shutdown_tx, + ); + } + fn sandbox_record(id: &str, name: &str, phase: SandboxPhase) -> Sandbox { Sandbox { id: id.to_string(), @@ -1417,6 +1546,122 @@ mod tests { ); } + #[tokio::test] + async fn apply_sandbox_update_promotes_connected_supervisor_session_to_ready() { + let runtime = test_runtime(Arc::new(TestDriver::default())).await; + let sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Provisioning); + runtime.store.put_message(&sandbox).await.unwrap(); + + register_test_supervisor_session(&runtime, "sb-1"); + + runtime + .apply_sandbox_update(DriverSandbox { + id: "sb-1".to_string(), + name: "sandbox-a".to_string(), + namespace: "default".to_string(), + spec: None, + status: Some(make_driver_status(make_driver_condition( + "Starting", + "VM is starting", + ))), + }) + .await + .unwrap(); + + let stored = runtime + .store + .get_message::("sb-1") + .await + .unwrap() + .unwrap(); + assert_eq!( + SandboxPhase::try_from(stored.phase).unwrap(), + SandboxPhase::Ready + ); + let ready = stored + .status + .as_ref() + .and_then(|status| { + status + .conditions + .iter() + .find(|condition| condition.r#type == "Ready") + }) + .unwrap(); + assert_eq!(ready.status, "True"); + assert_eq!(ready.reason, "DependenciesReady"); + assert_eq!(ready.message, "Supervisor session connected"); + } + + #[tokio::test] + async fn supervisor_session_connected_promotes_store_state_without_driver_refresh() { + let runtime = test_runtime(Arc::new(TestDriver::default())).await; + let sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Provisioning); + runtime.store.put_message(&sandbox).await.unwrap(); + + runtime.supervisor_session_connected("sb-1").await.unwrap(); + + let stored = runtime + .store + .get_message::("sb-1") + .await + .unwrap() + .unwrap(); + assert_eq!( + SandboxPhase::try_from(stored.phase).unwrap(), + SandboxPhase::Ready + ); + } + + #[tokio::test] + async fn supervisor_session_disconnected_demotes_ready_sandbox() { + let runtime = test_runtime(Arc::new(TestDriver::default())).await; + let mut sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Ready); + sandbox.status = Some(SandboxStatus { + sandbox_name: "sandbox-a".to_string(), + agent_pod: String::new(), + agent_fd: String::new(), + sandbox_fd: String::new(), + conditions: vec![SandboxCondition { + r#type: "Ready".to_string(), + status: "True".to_string(), + reason: "DependenciesReady".to_string(), + message: "Supervisor session connected".to_string(), + last_transition_time: String::new(), + }], + }); + runtime.store.put_message(&sandbox).await.unwrap(); + + runtime + .supervisor_session_disconnected("sb-1") + .await + .unwrap(); + + let stored = runtime + .store + .get_message::("sb-1") + .await + .unwrap() + .unwrap(); + assert_eq!( + SandboxPhase::try_from(stored.phase).unwrap(), + SandboxPhase::Provisioning + ); + let ready = stored + .status + .as_ref() + .and_then(|status| { + status + .conditions + .iter() + .find(|condition| condition.r#type == "Ready") + }) + .unwrap(); + assert_eq!(ready.status, "False"); + assert_eq!(ready.reason, "DependenciesNotReady"); + assert_eq!(ready.message, "Supervisor session disconnected"); + } + #[tokio::test] async fn reconcile_store_with_backend_applies_driver_snapshot() { let runtime = test_runtime(Arc::new(TestDriver { diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 9501ea3b2d..a40794037b 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -88,7 +88,7 @@ pub struct ServerState { pub settings_mutex: tokio::sync::Mutex<()>, /// Registry of active supervisor sessions and pending relay channels. - pub supervisor_sessions: supervisor_session::SupervisorSessionRegistry, + pub supervisor_sessions: Arc, } fn is_benign_tls_handshake_failure(error: &std::io::Error) -> bool { @@ -108,6 +108,7 @@ impl ServerState { sandbox_index: SandboxIndex, sandbox_watch_bus: SandboxWatchBus, tracing_log_bus: TracingLogBus, + supervisor_sessions: Arc, ) -> Self { Self { config, @@ -119,7 +120,7 @@ impl ServerState { ssh_connections_by_token: Mutex::new(HashMap::new()), ssh_connections_by_sandbox: Mutex::new(HashMap::new()), settings_mutex: tokio::sync::Mutex::new(()), - supervisor_sessions: supervisor_session::SupervisorSessionRegistry::new(), + supervisor_sessions, } } } @@ -150,6 +151,7 @@ pub async fn run_server( let sandbox_index = SandboxIndex::new(); let sandbox_watch_bus = SandboxWatchBus::new(); + let supervisor_sessions = Arc::new(supervisor_session::SupervisorSessionRegistry::new()); let compute = build_compute_runtime( &config, &vm_config, @@ -157,6 +159,7 @@ pub async fn run_server( sandbox_index.clone(), sandbox_watch_bus.clone(), tracing_log_bus.clone(), + supervisor_sessions.clone(), ) .await?; let state = Arc::new(ServerState::new( @@ -166,6 +169,7 @@ pub async fn run_server( sandbox_index, sandbox_watch_bus, tracing_log_bus, + supervisor_sessions, )); state.compute.spawn_watchers(); @@ -261,6 +265,7 @@ async fn build_compute_runtime( sandbox_index: SandboxIndex, sandbox_watch_bus: SandboxWatchBus, tracing_log_bus: TracingLogBus, + supervisor_sessions: Arc, ) -> Result { let driver = configured_compute_driver(config)?; info!(driver = %driver, "Using compute driver"); @@ -288,6 +293,7 @@ async fn build_compute_runtime( sandbox_index, sandbox_watch_bus, tracing_log_bus, + supervisor_sessions.clone(), ) .await .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}"))), @@ -300,6 +306,7 @@ async fn build_compute_runtime( sandbox_index, sandbox_watch_bus, tracing_log_bus, + supervisor_sessions, ) .await .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}"))) diff --git a/crates/openshell-server/src/supervisor_session.rs b/crates/openshell-server/src/supervisor_session.rs index f81ee9e3cb..d130bf71d0 100644 --- a/crates/openshell-server/src/supervisor_session.rs +++ b/crates/openshell-server/src/supervisor_session.rs @@ -180,6 +180,10 @@ impl SupervisorSessionRegistry { .map(|s| s.tx.clone()) } + pub fn has_session(&self, sandbox_id: &str) -> bool { + self.sessions.lock().unwrap().contains_key(sandbox_id) + } + fn pending_channel_ids(&self, sandbox_id: &str) -> Vec { self.pending_relays .lock() @@ -547,6 +551,19 @@ pub async fn handle_connect_supervisor( .await; } + if let Err(err) = state + .compute + .supervisor_session_connected(&sandbox_id) + .await + { + warn!( + sandbox_id = %sandbox_id, + session_id = %session_id, + error = %err, + "supervisor session: failed to mark sandbox ready" + ); + } + // Step 4: Spawn the session loop that reads inbound messages. let state_clone = Arc::clone(state); let sandbox_id_clone = sandbox_id.clone(); @@ -565,6 +582,18 @@ pub async fn handle_connect_supervisor( .remove_if_current(&sandbox_id_clone, &session_id); if still_ours { info!(sandbox_id = %sandbox_id_clone, session_id = %session_id, "supervisor session: ended"); + if let Err(err) = state_clone + .compute + .supervisor_session_disconnected(&sandbox_id_clone) + .await + { + warn!( + sandbox_id = %sandbox_id_clone, + session_id = %session_id, + error = %err, + "supervisor session: failed to mark sandbox disconnected" + ); + } } else { info!(sandbox_id = %sandbox_id_clone, session_id = %session_id, "supervisor session: ended (already superseded)"); } diff --git a/crates/openshell-vm/scripts/build-rootfs.sh b/crates/openshell-vm/scripts/build-rootfs.sh index d43046d4f4..566b32141e 100755 --- a/crates/openshell-vm/scripts/build-rootfs.sh +++ b/crates/openshell-vm/scripts/build-rootfs.sh @@ -119,6 +119,65 @@ verify_checksum() { fi } +ensure_build_nofile_limit() { + local desired="${OPENSHELL_VM_BUILD_NOFILE_LIMIT:-8192}" + local minimum=1024 + local current="" + local hard="" + local target="" + + [ "$(uname -s)" = "Darwin" ] || return 0 + command -v cargo-zigbuild >/dev/null 2>&1 || return 0 + + current="$(ulimit -n 2>/dev/null || echo "")" + case "${current}" in + ''|*[!0-9]*) + return 0 + ;; + esac + + if [ "${current}" -ge "${desired}" ]; then + return 0 + fi + + hard="$(ulimit -Hn 2>/dev/null || echo "")" + target="${desired}" + case "${hard}" in + ''|unlimited|infinity) + ;; + *[!0-9]*) + ;; + *) + if [ "${hard}" -lt "${target}" ]; then + target="${hard}" + fi + ;; + esac + + if [ "${target}" -gt "${current}" ] && ulimit -n "${target}" 2>/dev/null; then + echo "==> Raised open file limit for cargo-zigbuild: ${current} -> $(ulimit -n)" + fi + + current="$(ulimit -n 2>/dev/null || echo "${current}")" + case "${current}" in + ''|*[!0-9]*) + return 0 + ;; + esac + + if [ "${current}" -lt "${desired}" ]; then + echo "WARNING: Open file limit is ${current}; cargo-zigbuild is more reliable at ${desired}+ on macOS." + fi + + if [ "${current}" -lt "${minimum}" ]; then + echo "ERROR: Open file limit (${current}) is too low for cargo-zigbuild on macOS." + echo " Zig 0.14+ can fail with ProcessFdQuotaExceeded while linking large binaries." + echo " Run: ulimit -n ${desired}" + echo " Then re-run this script." + exit 1 + fi +} + if [ "$BASE_ONLY" = true ]; then echo "==> Building base openshell-vm rootfs" echo " Guest arch: ${GUEST_ARCH}" @@ -135,6 +194,10 @@ else fi echo "" +# cargo-zigbuild on macOS can exhaust the default per-process file descriptor +# limit while linking larger targets with Zig 0.14+. +ensure_build_nofile_limit + # ── Check for running VM ──────────────────────────────────────────────── # If an openshell-vm is using this rootfs via virtio-fs, wiping the rootfs # corrupts the VM's filesystem (e.g. /var disappears) causing cascading diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh index 5fd055036d..5990d8db62 100755 --- a/e2e/rust/e2e-vm.sh +++ b/e2e/rust/e2e-vm.sh @@ -2,245 +2,227 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# Run the Rust e2e smoke test against an openshell-vm gateway. +# Run the Rust e2e smoke test against an openshell-gateway running the +# standalone VM compute driver (`openshell-driver-vm`). # -# Usage: -# mise run e2e:vm # start new named VM on random port -# mise run e2e:vm -- --vm-port=30051 # reuse existing VM on port 30051 -# mise run e2e:vm -- --vm-port=30051 --vm-name=my-vm # reuse existing named VM and run exec check -# -# Options: -# --vm-port=PORT Skip VM startup and test against this port. -# --vm-name=NAME VM instance name. Auto-generated for fresh VMs. +# Architecture (post supervisor-initiated relay, PR #867): +# * The gateway never dials the sandbox. Instead, the in-guest +# supervisor opens an outbound `ConnectSupervisor` gRPC stream to +# the gateway on startup and keeps it alive for the sandbox +# lifetime. SSH (`/connect/ssh`) and `ExecSandbox` traffic ride the +# same TCP+TLS+HTTP/2 connection as multiplexed HTTP/2 streams. +# * There is no host-side SSH port forward. gvproxy still provides +# guest egress so the supervisor can reach the gateway, but it no +# longer forwards any TCP port back to the guest. +# * Readiness is authoritative on the gateway: a sandbox's phase +# flips to `Ready` the moment `ConnectSupervisor` registers, and +# back to `Provisioning` when the session drops. The VM driver +# only reports `Error` conditions for dead launcher processes. # -# When --vm-port is omitted: -# 1. Picks a random free host port -# 2. Starts the VM with --name --port :30051 -# 3. Waits for the VM to fully bootstrap (mTLS certs + gRPC health) -# 4. Verifies `openshell-vm exec` works -# 5. Runs the Rust smoke test -# 6. Tears down the VM +# Usage: +# mise run e2e:vm # -# When --vm-port is given the script assumes the VM is already running -# on that port and runs the smoke test. The VM exec check runs only when -# --vm-name is provided (so the script can target the correct instance). +# What the script does: +# 1. Ensures the VM runtime (libkrun + gvproxy + rootfs) is staged. +# 2. Builds `openshell-gateway`, `openshell-driver-vm`, and the +# `openshell` CLI with the embedded runtime. +# 3. On macOS, codesigns the VM driver (libkrun needs the +# `com.apple.security.hypervisor` entitlement). +# 4. Starts the gateway with `--drivers vm --disable-tls +# --disable-gateway-auth --db-url sqlite::memory:` on a random +# free port, waits for `Server listening`, then runs the +# cluster-agnostic Rust smoke test. +# 5. Tears the gateway down and (on failure) preserves the gateway +# log and every VM serial console log for post-mortem. # -# Prerequisites (when starting a new VM): `mise run vm:build` must already -# be done (the e2e:vm mise task handles this via depends). +# Prerequisites (handled automatically by this script if missing): +# - `mise run vm:setup` — downloads / builds the libkrun runtime. +# - `mise run vm:rootfs -- --base` — builds the sandbox rootfs tarball. set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -RUNTIME_DIR="${ROOT}/target/debug/openshell-vm.runtime" -GATEWAY_BIN="${ROOT}/target/debug/openshell-vm" -VM_GATEWAY_IMAGE="${IMAGE_REPO_BASE:-openshell}/gateway:${IMAGE_TAG:-dev}" -VM_GATEWAY_TAR_REL="var/lib/rancher/k3s/agent/images/openshell-server.tar.zst" -GUEST_PORT=30051 -TIMEOUT=180 - -named_vm_rootfs() { - local vm_version - - vm_version=$("${GATEWAY_BIN}" --version | awk '{print $2}') - printf '%s\n' "${XDG_DATA_HOME:-${HOME}/.local/share}/openshell/openshell-vm/${vm_version}/instances/${VM_NAME}/rootfs" -} - -vm_exec() { - local rootfs_args=() - if [ -n "${VM_ROOTFS_DIR:-}" ]; then - rootfs_args=(--rootfs "${VM_ROOTFS_DIR}") - fi - "${GATEWAY_BIN}" "${rootfs_args[@]}" --name "${VM_NAME}" exec -- "$@" -} +COMPRESSED_DIR="${ROOT}/target/vm-runtime-compressed" +GATEWAY_BIN="${ROOT}/target/debug/openshell-gateway" +DRIVER_BIN="${ROOT}/target/debug/openshell-driver-vm" + +# The VM driver places `compute-driver.sock` under --vm-driver-state-dir. +# AF_UNIX SUN_LEN is 104 bytes on macOS (108 on Linux), so paths anchored +# in the workspace's `target/` blow the limit on typical developer +# machines — e.g. a ~100-char `~/.superset/worktrees/.../target/...` +# prefix plus the `compute-driver.sock` leaf leaves no room. macOS' +# per-user `$TMPDIR` (`/var/folders/xx/.../T/`) can be 50+ chars too, +# so root state under `/tmp` unconditionally to keep UDS paths short. +STATE_DIR_ROOT="/tmp" + +# Smoke test timeouts. First boot extracts the embedded libkrun runtime +# (~60–90MB of zstd per architecture) and the sandbox rootfs (~200MB). +# The guest then runs k3s-free sandbox supervisor startup; a cold +# microVM is typically ready within ~15s. +GATEWAY_READY_TIMEOUT=60 +SANDBOX_PROVISION_TIMEOUT=180 + +# ── Build prerequisites ────────────────────────────────────────────── + +if [ ! -f "${COMPRESSED_DIR}/rootfs.tar.zst" ]; then + echo "==> Building base VM rootfs tarball (mise run vm:rootfs -- --base)" + mise run vm:rootfs -- --base +fi -prepare_named_vm_rootfs() { - if [ -z "${VM_NAME}" ]; then - return 0 - fi +if [ ! -f "${COMPRESSED_DIR}/rootfs.tar.zst" ] \ + || ! find "${COMPRESSED_DIR}" -maxdepth 1 -name 'libkrun*.zst' | grep -q .; then + echo "==> Preparing embedded VM runtime (mise run vm:setup)" + mise run vm:setup +fi - echo "Preparing named VM rootfs '${VM_NAME}'..." - VM_ROOTFS_DIR="$("${ROOT}/tasks/scripts/vm/ensure-vm-rootfs.sh" --name "${VM_NAME}" \ - | tail -n 1 | sed 's/^using openshell-vm rootfs at //')" - "${ROOT}/tasks/scripts/vm/sync-vm-rootfs.sh" --name "${VM_NAME}" -} +export OPENSHELL_VM_RUNTIME_COMPRESSED_DIR="${OPENSHELL_VM_RUNTIME_COMPRESSED_DIR:-${COMPRESSED_DIR}}" + +echo "==> Building openshell-gateway, openshell-driver-vm, openshell (CLI)" +cargo build \ + -p openshell-server \ + -p openshell-driver-vm \ + -p openshell-cli \ + --features openshell-core/dev-settings + +if [ "$(uname -s)" = "Darwin" ]; then + echo "==> Codesigning openshell-driver-vm (Hypervisor entitlement)" + codesign \ + --entitlements "${ROOT}/crates/openshell-driver-vm/entitlements.plist" \ + --force \ + -s - \ + "${DRIVER_BIN}" +fi -refresh_vm_gateway() { - if [ -z "${VM_NAME}" ]; then - return 0 +# ── Pick a random free host port for the gateway ───────────────────── + +HOST_PORT="$(python3 -c 'import socket +s = socket.socket() +s.bind(("", 0)) +print(s.getsockname()[1]) +s.close()')" + +# Per-run state dir so concurrent e2e runs don't collide on the UDS or +# sandbox state. The VM driver creates `/compute-driver.sock` +# and `/sandboxes//rootfs/` under here. Keep the +# basename short — see the SUN_LEN comment above. +RUN_STATE_DIR="${STATE_DIR_ROOT}/os-vm-e2e-${HOST_PORT}-$$" +mkdir -p "${RUN_STATE_DIR}" + +GATEWAY_LOG="$(mktemp /tmp/openshell-gateway-e2e.XXXXXX)" + +# ── Cleanup (trap) ─────────────────────────────────────────────────── + +cleanup() { + local exit_code=$? + + if [ -n "${GATEWAY_PID:-}" ] && kill -0 "${GATEWAY_PID}" 2>/dev/null; then + echo "Stopping openshell-gateway (pid ${GATEWAY_PID})..." + # SIGTERM first; gateway drops ManagedDriverProcess which SIGKILLs + # the driver and removes the UDS. Wait briefly, then force-kill. + kill -TERM "${GATEWAY_PID}" 2>/dev/null || true + for _ in 1 2 3 4 5 6 7 8 9 10; do + kill -0 "${GATEWAY_PID}" 2>/dev/null || break + sleep 0.5 + done + kill -KILL "${GATEWAY_PID}" 2>/dev/null || true + wait "${GATEWAY_PID}" 2>/dev/null || true fi - echo "Refreshing VM gateway StatefulSet image to ${VM_GATEWAY_IMAGE}..." - # Re-import the host-synced :dev image into the VM's containerd, then - # force a rollout when the StatefulSet already points at the same tag. - vm_exec sh -lc "set -eu; \ - image_tar='/${VM_GATEWAY_TAR_REL}'; \ - k3s ctr -n k8s.io images import \"\${image_tar}\" >/dev/null; \ - current_image=\$(kubectl -n openshell get statefulset/openshell -o jsonpath='{.spec.template.spec.containers[?(@.name==\"openshell\")].image}'); \ - if [ \"\${current_image}\" = \"${VM_GATEWAY_IMAGE}\" ]; then \ - kubectl -n openshell rollout restart statefulset/openshell >/dev/null; \ - else \ - kubectl -n openshell set image statefulset/openshell openshell=${VM_GATEWAY_IMAGE} >/dev/null; \ - fi; \ - kubectl -n openshell rollout status statefulset/openshell --timeout=300s" - echo "Gateway rollout complete." -} - -wait_for_gateway_health() { - local elapsed=0 timeout=60 consecutive_ok=0 - - echo "Waiting for refreshed gateway health..." - while [ "${elapsed}" -lt "${timeout}" ]; do - if "${ROOT}/target/debug/openshell" status >/dev/null 2>&1; then - consecutive_ok=$((consecutive_ok + 1)) - if [ "${consecutive_ok}" -ge 3 ]; then - echo "Gateway health confirmed after refresh." - return 0 - fi - else - consecutive_ok=0 - fi - - sleep 2 - elapsed=$((elapsed + 2)) - done - - echo "ERROR: refreshed gateway did not become healthy after ${timeout}s" - return 1 -} - -# ── Parse arguments ────────────────────────────────────────────────── -VM_PORT="" -VM_NAME="" -VM_ROOTFS_DIR="" -for arg in "$@"; do - case "$arg" in - --vm-port=*) VM_PORT="${arg#--vm-port=}" ;; - --vm-name=*) VM_NAME="${arg#--vm-name=}" ;; - *) echo "Unknown argument: $arg"; exit 1 ;; - esac -done + # On failure, keep the VM console log for debugging. We deliberately + # print it instead of leaving it on disk because the state dir gets + # wiped on success. + if [ "${exit_code}" -ne 0 ]; then + echo "=== gateway log (preserved for debugging) ===" + cat "${GATEWAY_LOG}" 2>/dev/null || true + echo "=== end gateway log ===" + + local console + while IFS= read -r -d '' console; do + echo "=== VM console log: ${console} ===" + cat "${console}" 2>/dev/null || true + echo "=== end VM console log ===" + done < <(find "${RUN_STATE_DIR}/sandboxes" -name 'rootfs-console.log' -print0 2>/dev/null) + fi -# ── Determine mode ─────────────────────────────────────────────────── -if [ -n "${VM_PORT}" ]; then - # Point at an already-running VM. - HOST_PORT="${VM_PORT}" - echo "Using existing VM on port ${HOST_PORT}." - if [ -n "${VM_NAME}" ]; then - prepare_named_vm_rootfs + rm -f "${GATEWAY_LOG}" 2>/dev/null || true + # Only wipe the per-run state dir on success. On failure, leave it for + # post-mortem (serial console logs, gvproxy logs, rootfs dumps). + if [ "${exit_code}" -eq 0 ]; then + rm -rf "${RUN_STATE_DIR}" 2>/dev/null || true + else + echo "NOTE: preserving ${RUN_STATE_DIR} for debugging" fi -else - # Pick a random free port and start a new VM. - HOST_PORT=$(python3 -c 'import socket; s=socket.socket(); s.bind(("",0)); print(s.getsockname()[1]); s.close()') - if [ -z "${VM_NAME}" ]; then - VM_NAME="e2e-${HOST_PORT}-$$" +} +trap cleanup EXIT + +# ── Launch the gateway + VM driver ─────────────────────────────────── + +SSH_HANDSHAKE_SECRET="$(openssl rand -hex 32)" + +echo "==> Starting openshell-gateway on 127.0.0.1:${HOST_PORT} (state: ${RUN_STATE_DIR})" + +# Pin --driver-dir to the workspace `target/debug/` so we always pick up +# the driver we just cargo-built. Without this, the gateway's +# `resolve_compute_driver_bin` fallback prefers +# `~/.local/libexec/openshell/openshell-driver-vm` when present +# (install-vm.sh installs there), which silently shadows development +# builds — a subtle source of stale-binary bugs in e2e runs. +"${GATEWAY_BIN}" \ + --drivers vm \ + --disable-tls \ + --disable-gateway-auth \ + --db-url 'sqlite::memory:' \ + --port "${HOST_PORT}" \ + --grpc-endpoint "http://127.0.0.1:${HOST_PORT}" \ + --ssh-handshake-secret "${SSH_HANDSHAKE_SECRET}" \ + --driver-dir "${ROOT}/target/debug" \ + --vm-driver-state-dir "${RUN_STATE_DIR}" \ + >"${GATEWAY_LOG}" 2>&1 & +GATEWAY_PID=$! + +# ── Wait for gateway readiness ─────────────────────────────────────── +# +# The gateway logs `INFO openshell_server: Server listening +# address=0.0.0.0:` after its tonic listener is up. That is the +# only signal the smoke test needs — the VM driver is spawned eagerly +# but sandboxes are created on demand, so "Server listening" is the +# right gate here. + +echo "==> Waiting for gateway readiness (timeout ${GATEWAY_READY_TIMEOUT}s)" +elapsed=0 +while ! grep -q 'Server listening' "${GATEWAY_LOG}" 2>/dev/null; do + if ! kill -0 "${GATEWAY_PID}" 2>/dev/null; then + echo "ERROR: openshell-gateway exited before becoming ready" + exit 1 fi - - cleanup() { - local exit_code=$? - if [ -n "${VM_PID:-}" ] && kill -0 "$VM_PID" 2>/dev/null; then - echo "Stopping openshell-vm (pid ${VM_PID})..." - kill "$VM_PID" 2>/dev/null || true - wait "$VM_PID" 2>/dev/null || true - fi - # On failure, preserve the VM console log for post-mortem debugging. - if [ "$exit_code" -ne 0 ] && [ -n "${VM_NAME:-}" ]; then - local console_log - console_log="$(named_vm_rootfs)-console.log" - if [ -f "$console_log" ]; then - echo "=== VM console log (preserved for debugging) ===" - cat "$console_log" - echo "=== end VM console log ===" - fi - fi - rm -f "${VM_LOG:-}" 2>/dev/null || true - if [ -n "${VM_NAME:-}" ]; then - rm -rf "$(dirname "$(named_vm_rootfs)")" 2>/dev/null || true - fi - } - trap cleanup EXIT - - prepare_named_vm_rootfs - - echo "Starting openshell-vm '${VM_NAME}' on port ${HOST_PORT}..." - if [ "$(uname -s)" = "Darwin" ]; then - export DYLD_FALLBACK_LIBRARY_PATH="${RUNTIME_DIR}${DYLD_FALLBACK_LIBRARY_PATH:+:${DYLD_FALLBACK_LIBRARY_PATH}}" + if [ "${elapsed}" -ge "${GATEWAY_READY_TIMEOUT}" ]; then + echo "ERROR: openshell-gateway did not become ready after ${GATEWAY_READY_TIMEOUT}s" + exit 1 fi + sleep 1 + elapsed=$((elapsed + 1)) +done - VM_LOG=$(mktemp /tmp/openshell-vm-e2e.XXXXXX) - rootfs_args=() - if [ -n "${VM_ROOTFS_DIR}" ]; then - rootfs_args=(--rootfs "${VM_ROOTFS_DIR}") - fi - "${GATEWAY_BIN}" "${rootfs_args[@]}" --name "${VM_NAME}" --port "${HOST_PORT}:${GUEST_PORT}" 2>"${VM_LOG}" & - VM_PID=$! - - # ── Wait for full bootstrap (mTLS certs + gRPC health) ───────────── - # The VM prints "Ready [Xs total]" to stderr after bootstrap_gateway() - # stores mTLS certs and wait_for_gateway_ready() confirms the gRPC - # service is responding. Waiting only for TCP port reachability (nc -z) - # is insufficient because port forwarding is established before the - # mTLS certs are written, causing `openshell status` to fail. - echo "Waiting for VM bootstrap to complete (timeout ${TIMEOUT}s)..." - elapsed=0 - while ! grep -q "^Ready " "${VM_LOG}" 2>/dev/null; do - if ! kill -0 "$VM_PID" 2>/dev/null; then - echo "ERROR: openshell-vm exited before becoming ready" - echo "VM log:" - cat "${VM_LOG}" - exit 1 - fi - if [ "$elapsed" -ge "$TIMEOUT" ]; then - echo "ERROR: openshell-vm did not become ready after ${TIMEOUT}s" - echo "VM log:" - cat "${VM_LOG}" - exit 1 - fi - sleep 2 - elapsed=$((elapsed + 2)) - done - echo "Gateway is ready (${elapsed}s)." - echo "VM log:" - cat "${VM_LOG}" -fi +echo "==> Gateway ready after ${elapsed}s" -# ── Exec into the VM (when instance name is known) ─────────────────── -if [ -n "${VM_NAME}" ]; then - echo "Verifying openshell-vm exec for '${VM_NAME}'..." - exec_elapsed=0 - exec_timeout=60 - until vm_exec /bin/true; do - if [ "$exec_elapsed" -ge "$exec_timeout" ]; then - echo "ERROR: openshell-vm exec did not become ready after ${exec_timeout}s" - exit 1 - fi - sleep 2 - exec_elapsed=$((exec_elapsed + 2)) - done - echo "VM exec succeeded." -else - echo "Skipping openshell-vm exec check (provide --vm-name for existing VMs)." -fi +# ── Run the smoke test ─────────────────────────────────────────────── +# +# The CLI takes OPENSHELL_GATEWAY_ENDPOINT directly; no gateway +# metadata lookup needed when TLS is disabled. -refresh_vm_gateway +export OPENSHELL_GATEWAY_ENDPOINT="http://127.0.0.1:${HOST_PORT}" -# ── Run the smoke test ─────────────────────────────────────────────── -# The openshell CLI reads OPENSHELL_GATEWAY_ENDPOINT to connect to the -# gateway directly, and OPENSHELL_GATEWAY to resolve mTLS certs from -# ~/.config/openshell/gateways//mtls/. -# In the VM, the overlayfs snapshotter re-extracts all image layers on -# every boot. The 1GB sandbox base image extraction can take >300s -# under contention, so allow 600s for sandbox provisioning. -export OPENSHELL_PROVISION_TIMEOUT=600 -export OPENSHELL_GATEWAY_ENDPOINT="https://127.0.0.1:${HOST_PORT}" -if [ -n "${VM_NAME}" ]; then - export OPENSHELL_GATEWAY="openshell-vm-${VM_NAME}" -else - export OPENSHELL_GATEWAY="openshell-vm" -fi +# The VM driver creates each sandbox VM from scratch — the embedded +# rootfs is extracted per sandbox, and the guest's sandbox supervisor +# then initializes policy, netns, Landlock, and sshd. On a cold host +# this is ~15s; allow 180s for slower CI runners. +export OPENSHELL_PROVISION_TIMEOUT="${SANDBOX_PROVISION_TIMEOUT}" -echo "Running e2e smoke test (gateway: ${OPENSHELL_GATEWAY}, endpoint: ${OPENSHELL_GATEWAY_ENDPOINT})..." -cargo build -p openshell-cli --features openshell-core/dev-settings -wait_for_gateway_health -cargo test --manifest-path e2e/rust/Cargo.toml --features e2e --test smoke -- --nocapture +echo "==> Running e2e smoke test (endpoint: ${OPENSHELL_GATEWAY_ENDPOINT})" +cargo test \ + --manifest-path "${ROOT}/e2e/rust/Cargo.toml" \ + --features e2e \ + --test smoke \ + -- --nocapture -echo "Smoke test passed." +echo "==> Smoke test passed." diff --git a/tasks/scripts/vm/smoke-orphan-cleanup.sh b/tasks/scripts/vm/smoke-orphan-cleanup.sh new file mode 100755 index 0000000000..9a37861a00 --- /dev/null +++ b/tasks/scripts/vm/smoke-orphan-cleanup.sh @@ -0,0 +1,204 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Smoke test: start the gateway with the VM driver, create a sandbox, then +# signal the gateway (SIGTERM then SIGKILL) and verify that no driver, +# launcher, gvproxy, or libkrun worker processes survive. +# +# Exit codes: +# 0 — both SIGTERM and SIGKILL cleanup passed +# 1 — one or more scenarios leaked survivors + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +cd "$ROOT" + +PORT="${OPENSHELL_SERVER_PORT:-8091}" +XDG="${TMPDIR:-/tmp}/vm-orphan-xdg-$$" +STATE_DIR="${TMPDIR:-/tmp}/openshell-vm-orphan-$$" +LOG="${TMPDIR:-/tmp}/vm-orphan-$$.log" + +cleanup_stray() { + # Best-effort: kill anything left over from our sandbox ids so repeated + # runs don't accumulate. + pkill -9 -f "openshell-vm-orphan-$$" 2>/dev/null || true + rm -rf "$XDG" "$STATE_DIR" 2>/dev/null || true + # Preserve the gateway log only on failure so operators can diagnose. + if [ "${EXIT_CODE:-0}" -ne 0 ]; then + echo "(log preserved at $LOG)" >&2 + else + rm -f "$LOG" "$LOG.create" 2>/dev/null || true + fi +} +trap cleanup_stray EXIT + +build_binaries() { + echo "==> Ensuring binaries are built" + if [ ! -x "$ROOT/target/debug/openshell-gateway" ] || [ ! -x "$ROOT/target/debug/openshell-driver-vm" ]; then + cargo build -p openshell-server -p openshell-driver-vm >&2 + fi + if [ "$(uname -s)" = "Darwin" ]; then + codesign \ + --entitlements "$ROOT/crates/openshell-driver-vm/entitlements.plist" \ + --force -s - \ + "$ROOT/target/debug/openshell-driver-vm" >/dev/null 2>&1 || true + fi +} + +start_gateway() { + local health_port=$((PORT + 1)) + echo "==> Starting gateway on port $PORT (state=$STATE_DIR, health=$health_port)" + mkdir -p "$STATE_DIR" + OPENSHELL_SERVER_PORT="$PORT" \ + OPENSHELL_HEALTH_PORT="$health_port" \ + OPENSHELL_DB_URL="sqlite:$STATE_DIR/openshell.db" \ + OPENSHELL_DRIVERS=vm \ + OPENSHELL_DRIVER_DIR="$ROOT/target/debug" \ + OPENSHELL_GRPC_ENDPOINT="http://host.containers.internal:$PORT" \ + OPENSHELL_SSH_GATEWAY_HOST=127.0.0.1 \ + OPENSHELL_SSH_GATEWAY_PORT="$PORT" \ + OPENSHELL_SSH_HANDSHAKE_SECRET=dev-vm-driver-secret \ + OPENSHELL_VM_DRIVER_STATE_DIR="$STATE_DIR" \ + OPENSHELL_VM_RUNTIME_COMPRESSED_DIR="$ROOT/target/vm-runtime-compressed" \ + nohup "$ROOT/target/debug/openshell-gateway" --disable-tls \ + > "$LOG" 2>&1 & + GATEWAY_PID=$! + echo "gateway pid=$GATEWAY_PID" + + for _ in $(seq 1 60); do + if grep -q "Server listening" "$LOG" 2>/dev/null; then + return 0 + fi + if ! kill -0 "$GATEWAY_PID" 2>/dev/null; then + echo "!! gateway died before ready" + tail -40 "$LOG" >&2 + return 1 + fi + sleep 1 + done + echo "!! gateway never reported ready" + tail -40 "$LOG" >&2 + return 1 +} + +create_sandbox() { + echo "==> Creating sandbox (--keep, long-running)" + mkdir -p "$XDG" + XDG_CONFIG_HOME="$XDG" "$ROOT/scripts/bin/openshell" gateway add \ + --name vm-orphan http://127.0.0.1:"$PORT" >/dev/null + XDG_CONFIG_HOME="$XDG" "$ROOT/scripts/bin/openshell" gateway select vm-orphan >/dev/null + + # Run the CLI in the background; it blocks waiting for sleep to finish. + XDG_CONFIG_HOME="$XDG" "$ROOT/scripts/bin/openshell" sandbox create \ + --name "orphan-$$" --keep -- sleep 99999 \ + > "$LOG.create" 2>&1 & + CLI_PID=$! + + for _ in $(seq 1 60); do + if pgrep -f "openshell-vm-orphan-$$|$STATE_DIR/sandboxes/" >/dev/null 2>&1; then + if pgrep -f gvproxy >/dev/null 2>&1; then + echo "sandbox came up (cli pid=$CLI_PID)" + return 0 + fi + fi + sleep 2 + done + echo "!! sandbox never came up" + tail -40 "$LOG" "$LOG.create" >&2 2>/dev/null || true + return 1 +} + +snapshot_kids() { + # Return all PIDs whose --state-dir or --vm-rootfs references our + # per-run directory, plus any gvproxy that mentions our socket base. + pgrep -fl "state-dir $STATE_DIR|$STATE_DIR/sandboxes" 2>/dev/null || true + pgrep -fl "gvproxy" 2>/dev/null | grep "osd-gv" || true +} + +count_alive() { + local alive + alive=$(pgrep -f "state-dir $STATE_DIR|$STATE_DIR/sandboxes" 2>/dev/null | wc -l | tr -d ' ') + local gv + gv=$(pgrep -f 'gvproxy' 2>/dev/null | xargs -r ps -o pid=,command= -p 2>/dev/null | grep -c 'osd-gv' || true) + echo $((alive + gv)) +} + +verify_cleanup() { + local label="$1" + local deadline="$2" + local waited=0 + while [ "$waited" -lt "$deadline" ]; do + local n + n=$(count_alive) + if [ "$n" = "0" ]; then + echo " PASS ($label): all descendants gone after ${waited}s" + return 0 + fi + sleep 1 + waited=$((waited + 1)) + done + echo " FAIL ($label): $(count_alive) descendants still alive after ${deadline}s:" + snapshot_kids | sed 's/^/ /' + return 1 +} + +run_scenario() { + local signal="$1" + local label="$2" + echo "======================================================" + echo "Scenario: $label (signal $signal)" + echo "======================================================" + + start_gateway || return 1 + create_sandbox || { kill -9 "$GATEWAY_PID" 2>/dev/null; return 1; } + + echo "-- process tree before signal --" + snapshot_kids | sed 's/^/ /' + echo + + echo "-> kill -$signal $GATEWAY_PID" + kill "-$signal" "$GATEWAY_PID" 2>/dev/null || true + + verify_cleanup "$label" 15 + local rc=$? + + # Belt-and-braces teardown between scenarios. + pkill -9 -f "$STATE_DIR/sandboxes|$STATE_DIR " 2>/dev/null || true + pkill -9 -f 'gvproxy.*osd-gv' 2>/dev/null || true + rm -rf "$STATE_DIR" /tmp/osd-gv "$XDG" 2>/dev/null || true + # CLI may still be running; reap it. + kill "${CLI_PID:-0}" 2>/dev/null || true + sleep 1 + + return $rc +} + +main() { + build_binaries + local overall=0 + + # Clean starting state. + pkill -9 -f 'openshell-gateway|openshell-driver-vm' 2>/dev/null || true + pkill -9 -f 'gvproxy.*osd-gv' 2>/dev/null || true + sleep 1 + + if ! run_scenario TERM "graceful SIGTERM"; then + overall=1 + fi + + if ! run_scenario KILL "abrupt SIGKILL"; then + overall=1 + fi + + if [ "$overall" -eq 0 ]; then + echo "ALL SCENARIOS PASSED" + else + echo "ONE OR MORE SCENARIOS FAILED" + fi + EXIT_CODE=$overall + return $overall +} + +main "$@" diff --git a/tasks/scripts/vm/vm-setup.sh b/tasks/scripts/vm/vm-setup.sh index e7ae06d082..bccb7f7542 100755 --- a/tasks/scripts/vm/vm-setup.sh +++ b/tasks/scripts/vm/vm-setup.sh @@ -128,4 +128,4 @@ echo " Compressed artifacts in: ${OUTPUT_DIR}" echo "" echo "Next steps:" echo " mise run vm:rootfs --base # build rootfs (requires Docker)" -echo " mise run vm # build and run the VM" +echo " mise run gateway:vm # start openshell-gateway with the VM driver" diff --git a/tasks/test.toml b/tasks/test.toml index f24ea6f2b8..cf45d2b6bc 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -49,6 +49,5 @@ env = { UV_NO_SYNC = "1", PYTHONPATH = "python" } run = "uv run pytest -o python_files='test_*.py' -m gpu -n ${E2E_PARALLEL:-1} e2e/python" ["e2e:vm"] -description = "Boot openshell-vm and run smoke e2e (macOS ARM64; pass -- --vm-port=N [--vm-name=NAME] to reuse)" -depends = ["build:docker:gateway", "vm:build"] +description = "Start openshell-gateway with the VM compute driver and run the cluster-agnostic smoke e2e" run = "e2e/rust/e2e-vm.sh" diff --git a/tasks/vm.toml b/tasks/vm.toml index ca06b08c13..0a44b4ff72 100644 --- a/tasks/vm.toml +++ b/tasks/vm.toml @@ -5,22 +5,25 @@ # # Workflow: # mise run vm:setup # one-time: download pre-built runtime (~30s) -# mise run vm # build + run the VM +# mise run gateway:vm # start openshell-gateway with the VM driver +# mise run vm # build + run the standalone openshell-vm microVM # mise run vm:clean # wipe everything and start over # -# See crates/openshell-vm/README.md for full documentation. +# See crates/openshell-driver-vm/README.md for the `gateway:vm` flow and +# crates/openshell-vm/README.md for the standalone microVM path. # ═══════════════════════════════════════════════════════════════════════════ # Main Commands # ═══════════════════════════════════════════════════════════════════════════ +["gateway:vm"] +description = "Build openshell-gateway + openshell-driver-vm and start the gateway with the VM driver" +run = "crates/openshell-driver-vm/start.sh" + [vm] -description = "Build and run the openshell-vm microVM" +description = "Build and run the standalone openshell-vm microVM" depends = ["build:docker:gateway"] -run = [ - "mise run vm:build", - "tasks/scripts/vm/run-vm.sh", -] +run = ["mise run vm:build", "tasks/scripts/vm/run-vm.sh"] ["vm:build"] description = "Build the openshell-vm binary with embedded runtime" @@ -42,3 +45,7 @@ run = "tasks/scripts/vm/build-rootfs-tarball.sh" ["vm:clean"] description = "Remove all VM cached artifacts (runtime, rootfs, builds)" run = "tasks/scripts/vm/vm-clean.sh" + +["vm:smoke:orphan-cleanup"] +description = "Smoke test: start gateway+driver, create a sandbox, signal the gateway, assert no orphaned processes survive" +run = "tasks/scripts/vm/smoke-orphan-cleanup.sh" From d0a29b64eb43b2d44bba5dbe105dad7c26f816f1 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Thu, 23 Apr 2026 08:27:27 -0700 Subject: [PATCH 030/142] fix(driver-vm): preflight supervisor cross-compile toolchain in start.sh (#931) --- crates/openshell-driver-vm/README.md | 3 ++ crates/openshell-driver-vm/start.sh | 33 +++++++++++++++++++++ crates/openshell-vm/README.md | 3 ++ crates/openshell-vm/scripts/build-rootfs.sh | 31 +++++++++++++------ 4 files changed, 61 insertions(+), 9 deletions(-) diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index 8808b25d90..0a82999d33 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -157,6 +157,9 @@ The VM guest's serial console is appended to `//console.l - macOS on Apple Silicon, or Linux on aarch64/x86_64 with KVM - Rust toolchain +- Guest-supervisor cross-compile toolchain (needed on macOS, and on Linux when host arch ≠ guest arch): + - Matching rustup target: `rustup target add aarch64-unknown-linux-gnu` (or `x86_64-unknown-linux-gnu` for an amd64 guest) + - `cargo install --locked cargo-zigbuild` and `brew install zig` (or distro equivalent). `build-rootfs.sh` uses `cargo zigbuild` to cross-compile the in-VM `openshell-sandbox` supervisor binary. - [mise](https://mise.jdx.dev/) task runner - Docker (needed by `mise run vm:rootfs` to build the base rootfs) - `gh` CLI (used by `mise run vm:setup` to download pre-built runtime artifacts) diff --git a/crates/openshell-driver-vm/start.sh b/crates/openshell-driver-vm/start.sh index b5aebbefd8..0579e8aa08 100755 --- a/crates/openshell-driver-vm/start.sh +++ b/crates/openshell-driver-vm/start.sh @@ -41,7 +41,40 @@ normalize_bool() { esac } +check_supervisor_cross_toolchain() { + # The sandbox supervisor inside the guest is always Linux. On non-Linux + # hosts (macOS) and on Linux hosts with a different arch than the guest, + # we cross-compile via cargo-zigbuild and need the matching rustup target. + local host_os host_arch guest_arch rust_target + host_os="$(uname -s)" + host_arch="$(uname -m)" + guest_arch="${GUEST_ARCH:-${host_arch}}" + case "${guest_arch}" in + arm64|aarch64) rust_target="aarch64-unknown-linux-gnu" ;; + x86_64|amd64) rust_target="x86_64-unknown-linux-gnu" ;; + *) return 0 ;; + esac + if [ "${host_os}" = "Linux" ] && [ "${host_arch}" = "${guest_arch}" ]; then + return 0 + fi + local missing=0 + if ! command -v cargo-zigbuild >/dev/null 2>&1; then + echo "ERROR: cargo-zigbuild not found (required to cross-compile the guest supervisor)." >&2 + echo " Install: cargo install --locked cargo-zigbuild && brew install zig" >&2 + missing=1 + fi + if ! rustup target list --installed 2>/dev/null | grep -qx "${rust_target}"; then + echo "ERROR: Rust target '${rust_target}' not installed." >&2 + echo " Install: rustup target add ${rust_target}" >&2 + missing=1 + fi + if [ "${missing}" -ne 0 ]; then + exit 1 + fi +} + if [ ! -f "${COMPRESSED_DIR}/rootfs.tar.zst" ]; then + check_supervisor_cross_toolchain echo "==> Building base VM rootfs tarball" mise run vm:rootfs -- --base fi diff --git a/crates/openshell-vm/README.md b/crates/openshell-vm/README.md index fcca20d5b9..b23cc3c271 100644 --- a/crates/openshell-vm/README.md +++ b/crates/openshell-vm/README.md @@ -18,6 +18,9 @@ mise run vm - **macOS (Apple Silicon)** or **Linux (aarch64 or x86_64 with KVM)** - Rust toolchain +- Guest-supervisor cross-compile toolchain (needed on macOS, and on Linux when host arch ≠ guest arch): + - Matching rustup target: `rustup target add aarch64-unknown-linux-gnu` (or `x86_64-unknown-linux-gnu` for an amd64 guest) + - `cargo install --locked cargo-zigbuild` and `brew install zig` (or distro equivalent). `build-rootfs.sh` uses `cargo zigbuild` to cross-compile the in-VM `openshell-sandbox` supervisor binary. - [mise](https://mise.jdx.dev/) task runner - Docker (for rootfs builds) - `gh` CLI (for downloading pre-built runtime) diff --git a/crates/openshell-vm/scripts/build-rootfs.sh b/crates/openshell-vm/scripts/build-rootfs.sh index 566b32141e..23834c30e4 100755 --- a/crates/openshell-vm/scripts/build-rootfs.sh +++ b/crates/openshell-vm/scripts/build-rootfs.sh @@ -365,16 +365,29 @@ SUPERVISOR_TARGET="${RUST_TARGET}" SUPERVISOR_BIN="${PROJECT_ROOT}/target/${SUPERVISOR_TARGET}/release/openshell-sandbox" echo "==> Building openshell-sandbox supervisor binary (${SUPERVISOR_TARGET})..." -if command -v cargo-zigbuild >/dev/null 2>&1; then - cargo zigbuild --release -p openshell-sandbox --target "${SUPERVISOR_TARGET}" \ - --manifest-path "${PROJECT_ROOT}/Cargo.toml" 2>&1 | tail -5 +SUPERVISOR_BUILD_LOG="$(mktemp -t openshell-supervisor-build.XXXXXX.log)" +run_supervisor_build() { + if command -v cargo-zigbuild >/dev/null 2>&1; then + cargo zigbuild --release -p openshell-sandbox --target "${SUPERVISOR_TARGET}" \ + --manifest-path "${PROJECT_ROOT}/Cargo.toml" + else + # Fallback: use plain cargo build when cargo-zigbuild is not available. + # This works for native builds (e.g. building x86_64 on x86_64) but + # will fail for true cross-compilation without a cross toolchain. + echo " cargo-zigbuild not found, falling back to cargo build..." + cargo build --release -p openshell-sandbox --target "${SUPERVISOR_TARGET}" \ + --manifest-path "${PROJECT_ROOT}/Cargo.toml" + fi +} +if run_supervisor_build >"${SUPERVISOR_BUILD_LOG}" 2>&1; then + tail -5 "${SUPERVISOR_BUILD_LOG}" + rm -f "${SUPERVISOR_BUILD_LOG}" else - # Fallback: use plain cargo build when cargo-zigbuild is not available. - # This works for native builds (e.g. building x86_64 on x86_64) but - # will fail for true cross-compilation without a cross toolchain. - echo " cargo-zigbuild not found, falling back to cargo build..." - cargo build --release -p openshell-sandbox --target "${SUPERVISOR_TARGET}" \ - --manifest-path "${PROJECT_ROOT}/Cargo.toml" 2>&1 | tail -5 + status=$? + echo "ERROR: supervisor build failed. Full output:" >&2 + cat "${SUPERVISOR_BUILD_LOG}" >&2 + echo " (log saved at ${SUPERVISOR_BUILD_LOG})" >&2 + exit "${status}" fi if [ ! -f "${SUPERVISOR_BIN}" ]; then From 89dd10bd4e431d017ea7babb6c25c1d1cc6955ac Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Thu, 23 Apr 2026 08:28:27 -0700 Subject: [PATCH 031/142] fix(ci): e2e gate must verify work actually ran, not just top-level success (#926) --- .github/workflows/e2e-gate-check.yml | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/.github/workflows/e2e-gate-check.yml b/.github/workflows/e2e-gate-check.yml index 81b71ce64e..5065663c17 100644 --- a/.github/workflows/e2e-gate-check.yml +++ b/.github/workflows/e2e-gate-check.yml @@ -84,18 +84,30 @@ jobs: exit 1 fi + run_id=$(jq -r '.id' <<< "$latest") status=$(jq -r '.status' <<< "$latest") conclusion=$(jq -r '.conclusion' <<< "$latest") - if [ "$conclusion" = "success" ]; then - echo "$WORKFLOW_FILE succeeded for $HEAD_SHA." - exit 0 - fi - if [ "$status" != "completed" ]; then echo "::error::$WORKFLOW_FILE is $status for $HEAD_SHA. This gate will re-evaluate on completion." exit 1 fi - echo "::error::$WORKFLOW_FILE concluded as $conclusion for $HEAD_SHA." - exit 1 + if [ "$conclusion" != "success" ]; then + echo "::error::$WORKFLOW_FILE concluded as $conclusion for $HEAD_SHA." + exit 1 + fi + + # Top-level success isn't enough: if `pr_metadata` gated downstream + # jobs out (label wasn't set at run time), only the gate job itself + # concludes `success` and the workflow still reports `success`. + # Require at least one non-gate job to have succeeded as proof the + # label was present when the workflow ran. + real_success=$(gh api "repos/$GH_REPO/actions/runs/$run_id/jobs" --jq '[.jobs[] | select(.conclusion == "success" and .name != "Resolve PR metadata")] | length') + if [ "$real_success" -lt 1 ]; then + echo "::error::$WORKFLOW_FILE run $run_id only ran the metadata gate — $REQUIRED_LABEL was not set when the workflow last executed. Re-run $WORKFLOW_FILE so the gate re-evaluates with the label present." + exit 1 + fi + + echo "$WORKFLOW_FILE run $run_id executed and succeeded for $HEAD_SHA ($real_success non-gate job(s) passed)." + exit 0 From c6f579279ea8615ad74789ede84335e7d0010b88 Mon Sep 17 00:00:00 2001 From: "John T. Myers" <9696606+johntmyers@users.noreply.github.com> Date: Thu, 23 Apr 2026 09:37:08 -0700 Subject: [PATCH 032/142] fix(ci): bump ci-image tooling versions to address vendored CVEs (#929) Bumps the CI image tool pins to clear High-severity container findings flagged by nSpect against ghcr.io/nvidia/openshell/ci. These tools vendor the Go modules reported in the nSpect tracker. Changes: - DOCKER_VERSION 29.3.1 -> 29.4.1 - BUILDX_VERSION v0.32.1 -> v0.33.0 (bundles buildkit v0.29.0, supersedes 0.28.1 fix) - GH_VERSION 2.74.1 -> 2.91.0 - MISE_VERSION v2026.3.13 -> v2026.4.18 Covers (via vendored deps in the above binaries): - GHSA-p77j-4mvh-x3m3 grpc-go authorization bypass - GHSA-4c29-8rgm-jvjj BuildKit malicious-frontend file escape - GHSA-4vrq-3vrq-g6gg BuildKit Git subdir access to restricted files - GHSA-9h8m-3fm2-qjrq OpenTelemetry Go SDK PATH hijacking - GHSA-92mm-2pjq-r785 hashicorp/go-getter arbitrary file reads - GHSA-78h2-9frx-2jm8 go-jose JWE decryption panic - GHSA-4qg8-fj49-pxjh sigstore timestamp-authority excessive memory - GHSA-x744-4wpc-v9h2 Moby AuthZ plugin bypass Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- deploy/docker/Dockerfile.ci | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/deploy/docker/Dockerfile.ci b/deploy/docker/Dockerfile.ci index b09c93e119..4750bc350b 100644 --- a/deploy/docker/Dockerfile.ci +++ b/deploy/docker/Dockerfile.ci @@ -8,8 +8,8 @@ FROM nvcr.io/nvidia/base/ubuntu:noble-20251013 -ARG DOCKER_VERSION=29.3.1 -ARG BUILDX_VERSION=v0.32.1 +ARG DOCKER_VERSION=29.4.1 +ARG BUILDX_VERSION=v0.33.0 ARG TARGETARCH ENV DEBIAN_FRONTEND=noninteractive @@ -55,7 +55,7 @@ RUN case "$TARGETARCH" in \ && chmod +x /usr/local/lib/docker/cli-plugins/docker-buildx # Install GitHub CLI used by install.sh and CI jobs -ARG GH_VERSION=2.74.1 +ARG GH_VERSION=2.91.0 RUN case "$TARGETARCH" in \ amd64) gh_arch=amd64 ;; \ arm64) gh_arch=arm64 ;; \ @@ -65,7 +65,7 @@ RUN case "$TARGETARCH" in \ | tar xz --strip-components=2 -C /usr/local/bin "gh_${GH_VERSION}_linux_${gh_arch}/bin/gh" # Install mise -ARG MISE_VERSION=v2026.3.13 +ARG MISE_VERSION=v2026.4.18 RUN curl https://mise.run | MISE_VERSION=$MISE_VERSION sh # Copy mise.toml and task includes, then install all tools via mise From c5d58552104d9a16158e6d9b62d8f7eee7df1f7d Mon Sep 17 00:00:00 2001 From: "John T. Myers" <9696606+johntmyers@users.noreply.github.com> Date: Thu, 23 Apr 2026 09:38:01 -0700 Subject: [PATCH 033/142] fix(ci): bump helm to 4.1.4 to address plugin vulnerabilities (#928) Clears four High-severity container findings in the CI image, which installs helm via mise.toml: - CVE-2026-35204 / GHSA-vmx8-mqv2-9gmg (plugin path traversal) - CVE-2026-35205 / GHSA-q5jf-9vfq-h4h7 (plugin .prov verification fails open) Both are fixed in helm 4.1.4. Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- mise.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mise.toml b/mise.toml index b045b563ca..785db69757 100644 --- a/mise.toml +++ b/mise.toml @@ -19,7 +19,7 @@ node = "24" kubectl = "1.35.1" uv = "0.10.2" protoc = "29.6" -helm = "4.1.1" +helm = "4.1.4" "ubi:mozilla/sccache" = { version = "0.14.0", matching = "sccache-v" } "ubi:anchore/syft" = { version = "1.42.3", matching = "syft_" } "ubi:EmbarkStudios/cargo-about" = "0.8.4" From 8405ceaa6f6136c81ca908565a75182f9d6a02eb Mon Sep 17 00:00:00 2001 From: Seth Jennings Date: Thu, 23 Apr 2026 11:49:18 -0500 Subject: [PATCH 034/142] fix(skills): remove --assignee @me from gh pr/issue create commands (#937) Most contributors cannot assign PRs or issues to themselves due to repository settings. Remove the --assignee flag from skill templates to prevent gh CLI errors. --- .agents/skills/build-from-issue/SKILL.md | 1 - .agents/skills/create-github-issue/SKILL.md | 1 - .agents/skills/create-github-pr/SKILL.md | 21 +-------------------- .agents/skills/fix-security-issue/SKILL.md | 1 - 4 files changed, 1 insertion(+), 23 deletions(-) diff --git a/.agents/skills/build-from-issue/SKILL.md b/.agents/skills/build-from-issue/SKILL.md index 4e73a61be3..7b906fc2f3 100644 --- a/.agents/skills/build-from-issue/SKILL.md +++ b/.agents/skills/build-from-issue/SKILL.md @@ -478,7 +478,6 @@ Create the PR: ```bash gh pr create \ --title "(): " \ - --assignee "@me" \ --body "$(cat <<'EOF' > **🏗️ build-from-issue-agent** diff --git a/.agents/skills/create-github-issue/SKILL.md b/.agents/skills/create-github-issue/SKILL.md index 15063d2c7f..b1311a297e 100644 --- a/.agents/skills/create-github-issue/SKILL.md +++ b/.agents/skills/create-github-issue/SKILL.md @@ -114,7 +114,6 @@ GitHub built-in issue types (`Bug`, `Feature`, `Task`) should come from the matc | `--title, -t` | Issue title (required) | | `--body, -b` | Issue description | | `--label, -l` | Add label (can use multiple times) | -| `--assignee, -a` | Assign to user | | `--milestone, -m` | Add to milestone | | `--project, -p` | Add to project | | `--web` | Open in browser after creation | diff --git a/.agents/skills/create-github-pr/SKILL.md b/.agents/skills/create-github-pr/SKILL.md index 63f8dd5ab6..9050db6de5 100644 --- a/.agents/skills/create-github-pr/SKILL.md +++ b/.agents/skills/create-github-pr/SKILL.md @@ -99,22 +99,6 @@ gh pr create --title "PR title" --body "PR description" - `refactor(models): simplify deployment logic` - `chore(ci): update Python version in pipeline` -## Required PR Fields - -Every PR **must** have: - -1. **Assignee** - Always assign to yourself - -## Assignee and Reviewer - -### Always Assign to Yourself - -**Every PR must be assigned to the user creating it.** Use the `--assignee` flag: - -```bash -gh pr create --title "Title" --assignee "@me" -``` - ### Link to an Issue Use `Closes #` in the body to auto-close the issue when merged: @@ -122,7 +106,6 @@ Use `Closes #` in the body to auto-close the issue when merged: ```bash gh pr create \ --title "Fix validation error for empty requests" \ - --assignee "@me" \ --body "Closes #123 ## Summary @@ -135,7 +118,7 @@ gh pr create \ For work-in-progress that's not ready for review: ```bash -gh pr create --draft --title "WIP: New feature" --assignee "@me" +gh pr create --draft --title "WIP: New feature" ``` ### With Labels @@ -185,7 +168,6 @@ Populate the testing checklist based on what was actually run. Check boxes for s ```bash gh pr create \ --title "feat(cli): add pagination to sandbox list" \ - --assignee "@me" \ --body "$(cat <<'EOF' ## Summary @@ -222,7 +204,6 @@ EOF | ------------------- | ------------------------------------------ | | `--title, -t` | PR title (use conventional commit format) | | `--body, -b` | PR description | -| `--assignee, -a` | Assign to user (use `@me` for yourself) | | `--reviewer, -r` | Request review from user | | `--draft` | Create as draft (WIP) | | `--label, -l` | Add label (can use multiple times) | diff --git a/.agents/skills/fix-security-issue/SKILL.md b/.agents/skills/fix-security-issue/SKILL.md index 7df9b4178c..22ffd42541 100644 --- a/.agents/skills/fix-security-issue/SKILL.md +++ b/.agents/skills/fix-security-issue/SKILL.md @@ -207,7 +207,6 @@ Create a PR that closes the security issue. Put the full fix summary in the PR d ```bash gh pr create \ --title "fix(security): " \ - --assignee "@me" \ --label "topic:security" \ --body "$(cat <<'EOF' > **🔧 security-fix-agent** From b19a3dc681abc01842dfc9161f4a271c24bbe2d4 Mon Sep 17 00:00:00 2001 From: Florent BENOIT Date: Thu, 23 Apr 2026 19:59:55 +0200 Subject: [PATCH 035/142] chore(mise): replace deprecated ubi: prefix by github: prefix (#923) --- mise.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mise.toml b/mise.toml index 785db69757..6f855fcf06 100644 --- a/mise.toml +++ b/mise.toml @@ -21,8 +21,8 @@ uv = "0.10.2" protoc = "29.6" helm = "4.1.4" "ubi:mozilla/sccache" = { version = "0.14.0", matching = "sccache-v" } -"ubi:anchore/syft" = { version = "1.42.3", matching = "syft_" } -"ubi:EmbarkStudios/cargo-about" = "0.8.4" +"github:anchore/syft" = { version = "1.42.3" } +"github:EmbarkStudios/cargo-about" = { version = "0.8.4", version_prefix = "" } zig = "0.14.1" [env] From 9bc2e2cc31c41ed230963112c6e88485e2eb81d9 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Thu, 23 Apr 2026 11:26:24 -0700 Subject: [PATCH 036/142] fix(ci): rename mise --no-prepare to --no-deps (#942) mise 2026.4.18 (bumped in #929) renamed the `--no-prepare` flag to `--no-deps`. Update workflow invocations to match. --- .github/workflows/docker-build.yml | 2 +- .github/workflows/e2e-gpu-test.yaml | 6 +++--- .github/workflows/e2e-test.yml | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 69a8d1d178..669fa9f198 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -93,4 +93,4 @@ jobs: # Enable dev-settings feature for test settings (dummy_bool, dummy_int) # used by e2e tests. EXTRA_CARGO_FEATURES: openshell-core/dev-settings - run: mise run --no-prepare build:docker:${{ inputs.component }} + run: mise run --no-deps build:docker:${{ inputs.component }} diff --git a/.github/workflows/e2e-gpu-test.yaml b/.github/workflows/e2e-gpu-test.yaml index 9feda37695..cde1f4792c 100644 --- a/.github/workflows/e2e-gpu-test.yaml +++ b/.github/workflows/e2e-gpu-test.yaml @@ -64,7 +64,7 @@ jobs: run: docker pull ghcr.io/nvidia/openshell/cluster:${{ inputs.image-tag }} - name: Install Python dependencies and generate protobuf stubs - run: uv sync --frozen && mise run --no-prepare python:proto + run: uv sync --frozen && mise run --no-deps python:proto - name: Bootstrap GPU cluster env: @@ -76,7 +76,7 @@ jobs: SKIP_IMAGE_PUSH: "1" SKIP_CLUSTER_IMAGE_BUILD: "1" OPENSHELL_CLUSTER_IMAGE: ghcr.io/nvidia/openshell/cluster:${{ inputs.image-tag }} - run: mise run --no-prepare --skip-deps cluster + run: mise run --no-deps --skip-deps cluster - name: Run tests - run: mise run --no-prepare --skip-deps e2e:python:gpu + run: mise run --no-deps --skip-deps e2e:python:gpu diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index a89f4508fc..052e77e6ab 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -29,11 +29,11 @@ jobs: - suite: python cluster: e2e-python port: "8080" - cmd: "mise run --no-prepare --skip-deps e2e:python" + cmd: "mise run --no-deps --skip-deps e2e:python" - suite: rust cluster: e2e-rust port: "8081" - cmd: "mise run --no-prepare --skip-deps e2e:rust" + cmd: "mise run --no-deps --skip-deps e2e:rust" - suite: gateway-resume cluster: e2e-resume port: "8082" @@ -66,7 +66,7 @@ jobs: - name: Install Python dependencies and generate protobuf stubs if: matrix.suite == 'python' - run: uv sync --frozen && mise run --no-prepare python:proto + run: uv sync --frozen && mise run --no-deps python:proto - name: Build Rust CLI if: matrix.suite != 'python' @@ -84,7 +84,7 @@ jobs: SKIP_IMAGE_PUSH: "1" SKIP_CLUSTER_IMAGE_BUILD: "1" OPENSHELL_CLUSTER_IMAGE: ghcr.io/nvidia/openshell/cluster:${{ inputs.image-tag }} - run: mise run --no-prepare --skip-deps cluster + run: mise run --no-deps --skip-deps cluster - name: Run tests run: ${{ matrix.cmd }} From 3b7d30934e2c5805cc6f8d185a2fb0b5556d56bd Mon Sep 17 00:00:00 2001 From: Seth Jennings Date: Thu, 23 Apr 2026 14:53:13 -0500 Subject: [PATCH 037/142] feat(server): add Prometheus metrics infrastructure and gRPC/HTTP request metrics (#920) * feat(server): add Prometheus metrics infrastructure and gRPC/HTTP request metrics Add metrics exposition via a dedicated server port (--metrics-port, default disabled) following the same optional-listener pattern as the health port. The metrics crate facade records counters and histograms in the MultiplexedService hot path, and a PrometheusHandle renders them at GET /metrics on the dedicated port. Metrics added: - openshell_server_grpc_requests_total (counter: method, code) - openshell_server_grpc_request_duration_seconds (histogram: method, code) - openshell_server_http_requests_total (counter: path, status) - openshell_server_http_request_duration_seconds (histogram: path, status) * feat(helm): add metrics port to openshell-server helm chart Expose the Prometheus metrics endpoint in the helm chart by adding service.metricsPort (default 9090) to values, the --metrics-port arg and container port to the statefulset, and a metrics service port. Set metricsPort to 0 to disable. --- Cargo.lock | 113 +++++++++++++++- Cargo.toml | 4 + crates/openshell-core/src/config.rs | 13 ++ crates/openshell-server/Cargo.toml | 4 + crates/openshell-server/src/cli.rs | 23 ++++ crates/openshell-server/src/http.rs | 14 +- crates/openshell-server/src/lib.rs | 28 +++- crates/openshell-server/src/multiplex.rs | 124 +++++++++++++++++- deploy/helm/openshell/templates/service.yaml | 6 + .../helm/openshell/templates/statefulset.yaml | 9 ++ deploy/helm/openshell/values.yaml | 1 + 11 files changed, 335 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2d0bc6ce22..1253ffffc8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -892,6 +892,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-queue" version = "0.3.12" @@ -1455,6 +1464,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -1721,7 +1736,16 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash 0.2.0", ] [[package]] @@ -2721,6 +2745,52 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "metrics" +version = "0.24.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d5312e9ba3771cfa961b585728215e3d972c950a3eed9252aa093d6301277e8" +dependencies = [ + "ahash", + "portable-atomic", +] + +[[package]] +name = "metrics-exporter-prometheus" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3589659543c04c7dc5526ec858591015b87cd8746583b51b48ef4353f99dbcda" +dependencies = [ + "base64 0.22.1", + "http-body-util", + "hyper", + "hyper-util", + "indexmap 2.14.0", + "ipnet", + "metrics", + "metrics-util", + "quanta", + "thiserror 2.0.18", + "tokio", + "tracing", +] + +[[package]] +name = "metrics-util" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdfb1365fea27e6dd9dc1dbc19f570198bc86914533ad639dae939635f096be4" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.16.1", + "metrics", + "quanta", + "rand 0.9.2", + "rand_xoshiro", + "sketches-ddsketch", +] + [[package]] name = "miette" version = "7.6.0" @@ -3238,6 +3308,8 @@ dependencies = [ "hyper-rustls", "hyper-util", "ipnet", + "metrics", + "metrics-exporter-prometheus", "miette", "openshell-core", "openshell-driver-kubernetes", @@ -3857,6 +3929,21 @@ dependencies = [ "autotools", ] +[[package]] +name = "quanta" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + [[package]] name = "quinn" version = "0.11.9" @@ -3998,6 +4085,15 @@ version = "0.10.0-rc-3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f66ee92bc15280519ef199a274fe0cafff4245d31bc39aaa31c011ad56cb1f05" +[[package]] +name = "rand_xoshiro" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" +dependencies = [ + "rand_core 0.9.5", +] + [[package]] name = "ratatui" version = "0.26.3" @@ -4018,6 +4114,15 @@ dependencies = [ "unicode-width 0.1.14", ] +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags", +] + [[package]] name = "rcgen" version = "0.13.2" @@ -4790,6 +4895,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +[[package]] +name = "sketches-ddsketch" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b" + [[package]] name = "slab" version = "0.4.12" diff --git a/Cargo.toml b/Cargo.toml index b51aee3c25..cffad2cc1d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -58,6 +58,10 @@ tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } tracing-appender = "0.2" +# Metrics +metrics = "0.24" +metrics-exporter-prometheus = { version = "0.18", default-features = false, features = ["http-listener"] } + # Unix/Process nix = { version = "0.29", features = ["signal", "process", "user", "fs", "term"] } diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index e572537391..d9440745d3 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -63,6 +63,12 @@ pub struct Config { #[serde(default)] pub health_bind_address: Option, + /// Address to bind the Prometheus metrics endpoint to. + /// + /// When `None`, the dedicated metrics listener is disabled. + #[serde(default)] + pub metrics_bind_address: Option, + /// Log level (trace, debug, info, warn, error). #[serde(default = "default_log_level")] pub log_level: String, @@ -183,6 +189,7 @@ impl Config { Self { bind_address: default_bind_address(), health_bind_address: None, + metrics_bind_address: None, log_level: default_log_level(), tls, database_url: String::new(), @@ -216,6 +223,12 @@ impl Config { self } + #[must_use] + pub const fn with_metrics_bind_address(mut self, addr: SocketAddr) -> Self { + self.metrics_bind_address = Some(addr); + self + } + /// Create a new configuration with the given log level. #[must_use] pub fn with_log_level(mut self, level: impl Into) -> Self { diff --git a/crates/openshell-server/Cargo.toml b/crates/openshell-server/Cargo.toml index b2524ff0b2..6f8c949c78 100644 --- a/crates/openshell-server/Cargo.toml +++ b/crates/openshell-server/Cargo.toml @@ -56,6 +56,10 @@ anyhow = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } +# Metrics +metrics = { workspace = true } +metrics-exporter-prometheus = { workspace = true } + # Utilities futures = { workspace = true } bytes = { workspace = true } diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index 796068d216..1ed5a59828 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -28,6 +28,11 @@ struct Args { #[arg(long, default_value_t = 0, env = "OPENSHELL_HEALTH_PORT")] health_port: u16, + /// Port for the Prometheus metrics endpoint (/metrics). + /// Set to 0 to disable the dedicated metrics listener. + #[arg(long, default_value_t = 0, env = "OPENSHELL_METRICS_PORT")] + metrics_port: u16, + /// Log level (trace, debug, info, warn, error). #[arg(long, default_value = "info", env = "OPENSHELL_LOG_LEVEL")] log_level: String, @@ -202,6 +207,7 @@ async fn run_from_args(args: Args) -> Result<()> { ); let bind = SocketAddr::from(([0, 0, 0, 0], args.port)); + let tls = if args.disable_tls { None } else { @@ -241,6 +247,23 @@ async fn run_from_args(args: Args) -> Result<()> { config = config.with_health_bind_address(health_bind); } + if args.metrics_port != 0 { + if args.port == args.metrics_port { + return Err(miette::miette!( + "--port and --metrics-port must be different (both set to {})", + args.port + )); + } + if args.health_port != 0 && args.health_port == args.metrics_port { + return Err(miette::miette!( + "--health-port and --metrics-port must be different (both set to {})", + args.health_port + )); + } + let metrics_bind = SocketAddr::from(([0, 0, 0, 0], args.metrics_port)); + config = config.with_metrics_bind_address(metrics_bind); + } + config = config .with_database_url(args.db_url) .with_compute_drivers(args.drivers) diff --git a/crates/openshell-server/src/http.rs b/crates/openshell-server/src/http.rs index 4d45c53521..7650c2339a 100644 --- a/crates/openshell-server/src/http.rs +++ b/crates/openshell-server/src/http.rs @@ -3,7 +3,8 @@ //! HTTP health endpoints using Axum. -use axum::{Json, Router, http::StatusCode, response::IntoResponse, routing::get}; +use axum::{Json, Router, extract::State, http::StatusCode, response::IntoResponse, routing::get}; +use metrics_exporter_prometheus::PrometheusHandle; use serde::Serialize; use std::sync::Arc; @@ -45,6 +46,17 @@ pub fn health_router() -> Router { .route("/readyz", get(readyz)) } +/// Create the metrics router for the dedicated metrics port. +pub fn metrics_router(handle: PrometheusHandle) -> Router { + Router::new() + .route("/metrics", get(render_metrics)) + .with_state(handle) +} + +async fn render_metrics(State(handle): State) -> impl IntoResponse { + handle.render() +} + /// Create the HTTP router. pub fn http_router(state: Arc) -> Router { crate::ssh_tunnel::router(state.clone()) diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index a40794037b..0442640d46 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -35,6 +35,7 @@ mod tls; pub mod tracing_bus; mod ws_tunnel; +use metrics_exporter_prometheus::PrometheusBuilder; use openshell_core::{ComputeDriverKind, Config, Error, Result}; use std::collections::HashMap; use std::io::ErrorKind; @@ -45,7 +46,7 @@ use tracing::{debug, error, info}; use compute::{ComputeRuntime, VmComputeConfig}; pub use grpc::OpenShellService; -pub use http::{health_router, http_router}; +pub use http::{health_router, http_router, metrics_router}; pub use multiplex::{MultiplexService, MultiplexedService}; use openshell_driver_kubernetes::KubernetesComputeConfig; use persistence::Store; @@ -205,6 +206,31 @@ pub async fn run_server( info!("Health server disabled"); } + // Bind the Prometheus metrics endpoint on a dedicated port when configured. + if let Some(metrics_bind_address) = config.metrics_bind_address { + let prometheus_handle = PrometheusBuilder::new() + .install_recorder() + .map_err(|e| Error::config(format!("failed to install metrics recorder: {e}")))?; + let metrics_listener = TcpListener::bind(metrics_bind_address).await.map_err(|e| { + Error::transport(format!( + "failed to bind metrics port {metrics_bind_address}: {e}", + )) + })?; + info!(address = %metrics_bind_address, "Metrics server listening"); + tokio::spawn(async move { + if let Err(e) = axum::serve( + metrics_listener, + metrics_router(prometheus_handle).into_make_service(), + ) + .await + { + error!("Metrics server error: {e}"); + } + }); + } else { + info!("Metrics server disabled"); + } + // Build TLS acceptor when TLS is configured; otherwise serve plaintext. let tls_acceptor = if let Some(tls) = &config.tls { Some(TlsAcceptor::from_files( diff --git a/crates/openshell-server/src/multiplex.rs b/crates/openshell-server/src/multiplex.rs index 841aa92630..e0c1599588 100644 --- a/crates/openshell-server/src/multiplex.rs +++ b/crates/openshell-server/src/multiplex.rs @@ -15,6 +15,7 @@ use hyper_util::{ rt::{TokioExecutor, TokioIo}, server::conn::auto::Builder, }; +use metrics::{counter, histogram}; use openshell_core::proto::{ inference_server::InferenceServer, open_shell_server::OpenShellServer, }; @@ -22,7 +23,7 @@ use std::future::Future; use std::pin::Pin; use std::sync::Arc; use std::task::{Context, Poll}; -use std::time::Duration; +use std::time::{Duration, Instant}; use tokio::io::{AsyncRead, AsyncWrite}; use tower::{ServiceBuilder, ServiceExt}; use tower_http::trace::TraceLayer; @@ -192,6 +193,8 @@ where .is_some_and(|v| v.as_bytes().starts_with(b"application/grpc")); if is_grpc { + let method = grpc_method_from_path(req.uri().path()); + let start = Instant::now(); let mut grpc = self.grpc.clone(); Box::pin(async move { let (parts, body) = req.into_parts(); @@ -206,11 +209,18 @@ where .await .map_err(Into::into)?; + let code = grpc_status_from_response(&res); + let elapsed = start.elapsed().as_secs_f64(); + counter!("openshell_server_grpc_requests_total", "method" => method.clone(), "code" => code.clone()).increment(1); + histogram!("openshell_server_grpc_request_duration_seconds", "method" => method, "code" => code).record(elapsed); + let (parts, body) = res.into_parts(); let body = body.map_err(Into::into).boxed_unsync(); Ok(Response::from_parts(parts, BoxBody(body))) }) } else { + let path = normalize_http_path(req.uri().path()); + let start = Instant::now(); let mut http = self.http.clone(); Box::pin(async move { let (parts, body) = req.into_parts(); @@ -225,6 +235,11 @@ where .await .map_err(Into::into)?; + let status = res.status().as_u16().to_string(); + let elapsed = start.elapsed().as_secs_f64(); + counter!("openshell_server_http_requests_total", "path" => path, "status" => status.clone()).increment(1); + histogram!("openshell_server_http_request_duration_seconds", "path" => path, "status" => status).record(elapsed); + let (parts, body) = res.into_parts(); let body = body.map_err(Into::into).boxed_unsync(); Ok(Response::from_parts(parts, BoxBody(body))) @@ -258,6 +273,26 @@ fn log_response(res: &Response, latency: Duration, _span: &Span) { ); } +fn grpc_method_from_path(path: &str) -> String { + path.rsplit('/').next().unwrap_or(path).to_string() +} + +fn grpc_status_from_response(res: &Response) -> String { + res.headers() + .get("grpc-status") + .and_then(|v| v.to_str().ok()) + .map_or_else(|| "0".to_string(), ToString::to_string) +} + +fn normalize_http_path(path: &str) -> &'static str { + match path { + p if p.starts_with("/connect/ssh") => "/connect/ssh", + p if p.starts_with("/_ws_tunnel") => "/_ws_tunnel", + p if p.starts_with("/auth/") => "/auth", + _ => "unknown", + } +} + /// Boxed body type for uniform handling. pub struct BoxBody( http_body_util::combinators::UnsyncBoxBody>, @@ -282,3 +317,90 @@ impl Body for BoxBody { self.0.size_hint() } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn grpc_method_extracts_last_segment() { + assert_eq!( + grpc_method_from_path("/openshell.v1.OpenShell/CreateSandbox"), + "CreateSandbox" + ); + } + + #[test] + fn grpc_method_extracts_inference_service() { + assert_eq!( + grpc_method_from_path("/openshell.inference.v1.Inference/GetInferenceBundle"), + "GetInferenceBundle" + ); + } + + #[test] + fn grpc_method_handles_bare_path() { + assert_eq!(grpc_method_from_path("Health"), "Health"); + } + + #[test] + fn grpc_method_handles_single_slash() { + assert_eq!(grpc_method_from_path("/"), ""); + } + + #[test] + fn grpc_method_handles_empty_string() { + assert_eq!(grpc_method_from_path(""), ""); + } + + #[test] + fn normalize_ssh_path() { + assert_eq!(normalize_http_path("/connect/ssh"), "/connect/ssh"); + } + + #[test] + fn normalize_ssh_path_with_trailing_segments() { + assert_eq!( + normalize_http_path("/connect/ssh?token=abc"), + "/connect/ssh" + ); + } + + #[test] + fn normalize_ws_tunnel() { + assert_eq!(normalize_http_path("/_ws_tunnel"), "/_ws_tunnel"); + } + + #[test] + fn normalize_ws_tunnel_with_trailing() { + assert_eq!(normalize_http_path("/_ws_tunnel/foo"), "/_ws_tunnel"); + } + + #[test] + fn normalize_auth_path() { + assert_eq!(normalize_http_path("/auth/connect"), "/auth"); + } + + #[test] + fn normalize_auth_with_query() { + assert_eq!( + normalize_http_path("/auth/connect?callback_port=12345&code=AB7-X9KM"), + "/auth" + ); + } + + #[test] + fn normalize_unknown_path_collapses_to_unknown() { + assert_eq!(normalize_http_path("/random/scanner/probe"), "unknown"); + } + + #[test] + fn normalize_empty_path() { + assert_eq!(normalize_http_path(""), "unknown"); + } + + #[test] + fn normalize_root_path() { + assert_eq!(normalize_http_path("/"), "unknown"); + } +} diff --git a/deploy/helm/openshell/templates/service.yaml b/deploy/helm/openshell/templates/service.yaml index 0e4aa4b763..ebad42eab1 100644 --- a/deploy/helm/openshell/templates/service.yaml +++ b/deploy/helm/openshell/templates/service.yaml @@ -18,5 +18,11 @@ spec: {{- if and (eq .Values.service.type "NodePort") .Values.service.nodePort }} nodePort: {{ .Values.service.nodePort }} {{- end }} + {{- if .Values.service.metricsPort }} + - port: {{ .Values.service.metricsPort }} + targetPort: metrics + protocol: TCP + name: metrics + {{- end }} selector: {{- include "openshell.selectorLabels" . | nindent 4 }} diff --git a/deploy/helm/openshell/templates/statefulset.yaml b/deploy/helm/openshell/templates/statefulset.yaml index 8262511344..37ebcae80e 100644 --- a/deploy/helm/openshell/templates/statefulset.yaml +++ b/deploy/helm/openshell/templates/statefulset.yaml @@ -51,6 +51,10 @@ spec: - {{ .Values.service.port | quote }} - --health-port - {{ .Values.service.healthPort | quote }} + {{- if .Values.service.metricsPort }} + - --metrics-port + - {{ .Values.service.metricsPort | quote }} + {{- end }} - --log-level - {{ .Values.server.logLevel }} - --db-url @@ -118,6 +122,11 @@ spec: - name: health containerPort: {{ .Values.service.healthPort }} protocol: TCP + {{- if .Values.service.metricsPort }} + - name: metrics + containerPort: {{ .Values.service.metricsPort }} + protocol: TCP + {{- end }} startupProbe: httpGet: path: /healthz diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 2b8cc4e6ff..4ac8ab43a6 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -38,6 +38,7 @@ service: port: 8080 nodePort: 30051 healthPort: 8081 + metricsPort: 9090 # Pod restart behavior and health probe tuning. podLifecycle: From ab3f3e033df651f7af7fcc92acc1703d4086a877 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Thu, 23 Apr 2026 13:02:56 -0700 Subject: [PATCH 038/142] fix(ci): post E2E Gate check to the PR when workflow_run fires (#938) --- .github/workflows/e2e-gate.yml | 42 +++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/.github/workflows/e2e-gate.yml b/.github/workflows/e2e-gate.yml index 757a954662..67959fa8dc 100644 --- a/.github/workflows/e2e-gate.yml +++ b/.github/workflows/e2e-gate.yml @@ -10,13 +10,11 @@ on: permissions: {} jobs: + # On PR events, actually evaluate the gate — these runs post their check + # result to the PR's head SHA, which is what branch protection sees. e2e: name: E2E - # Run on every PR event; on workflow_run, only when the upstream was the - # matching gated workflow. - if: >- - github.event_name == 'pull_request' || - github.event.workflow_run.name == 'Branch E2E Checks' + if: github.event_name == 'pull_request' permissions: contents: read pull-requests: read @@ -28,9 +26,7 @@ jobs: gpu: name: GPU E2E - if: >- - github.event_name == 'pull_request' || - github.event.workflow_run.name == 'GPU Test' + if: github.event_name == 'pull_request' permissions: contents: read pull-requests: read @@ -39,3 +35,33 @@ jobs: with: required_label: test:e2e-gpu workflow_file: test-gpu.yml + + # When the guarded workflow finishes, GitHub fires `workflow_run` in the + # default-branch context — any check posted from here would land on `main`, + # not on the PR. Instead, find the latest `pull_request`-triggered gate run + # for the same head SHA and ask the API to re-run it. The re-run replays the + # original event (labels, head SHA) so the check posts to the PR again, and + # this time the gate sees the successful upstream run. + rerun-on-completion: + name: Re-run gate in PR context + if: github.event_name == 'workflow_run' + runs-on: ubuntu-latest + permissions: + actions: write + steps: + - name: Rerun latest PR-context gate for this SHA + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + shell: bash + run: | + set -euo pipefail + run_id=$(gh api "repos/$GH_REPO/actions/workflows/e2e-gate.yml/runs?event=pull_request&head_sha=$HEAD_SHA" \ + --jq '.workflow_runs | sort_by(.created_at) | reverse | .[0].id // empty') + if [ -z "$run_id" ]; then + echo "No pull_request-triggered E2E Gate run found for $HEAD_SHA — nothing to re-run." + exit 0 + fi + echo "Re-running E2E Gate run $run_id so it re-evaluates and posts a fresh check on the PR." + gh api --method POST "repos/$GH_REPO/actions/runs/$run_id/rerun" From 550c6e46cb7bc6dceb05eb8e7988d4775fb63ee6 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Thu, 23 Apr 2026 13:52:27 -0700 Subject: [PATCH 039/142] chore(helm): remove unused ClusterRole and ClusterRoleBinding (#943) * chore(helm): remove unused cluster rule from helm chart --- .../helm/openshell/templates/clusterrole.yaml | 24 ------------------- .../templates/clusterrolebinding.yaml | 17 ------------- 2 files changed, 41 deletions(-) delete mode 100644 deploy/helm/openshell/templates/clusterrole.yaml delete mode 100644 deploy/helm/openshell/templates/clusterrolebinding.yaml diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml deleted file mode 100644 index 2f98784583..0000000000 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "openshell.fullname" . }}-sandbox-runtimeclass - labels: - {{- include "openshell.labels" . | nindent 4 }} -rules: - - apiGroups: - - node.k8s.io - resources: - - runtimeclasses - verbs: - - get - - list - - apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list diff --git a/deploy/helm/openshell/templates/clusterrolebinding.yaml b/deploy/helm/openshell/templates/clusterrolebinding.yaml deleted file mode 100644 index 9dda1d7da4..0000000000 --- a/deploy/helm/openshell/templates/clusterrolebinding.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "openshell.fullname" . }}-sandbox-runtimeclass - labels: - {{- include "openshell.labels" . | nindent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "openshell.fullname" . }}-sandbox-runtimeclass -subjects: - - kind: ServiceAccount - name: {{ include "openshell.serviceAccountName" . }} - namespace: {{ .Release.Namespace }} From 0a09404c1d8248d19e8466a9c0940ba9dd7a7bd3 Mon Sep 17 00:00:00 2001 From: jtoelke2 <149006449+jtoelke2@users.noreply.github.com> Date: Thu, 23 Apr 2026 16:51:47 -0500 Subject: [PATCH 040/142] feat(ci): add shadow-shared-cpu-spike workflow for OS-49 Phase 2 (#934) Signed-off-by: Jonas Toelke --- .github/workflows/shadow-shared-cpu-spike.yml | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 .github/workflows/shadow-shared-cpu-spike.yml diff --git a/.github/workflows/shadow-shared-cpu-spike.yml b/.github/workflows/shadow-shared-cpu-spike.yml new file mode 100644 index 0000000000..6dae8ad9d0 --- /dev/null +++ b/.github/workflows/shadow-shared-cpu-spike.yml @@ -0,0 +1,57 @@ +name: Shadow — Shared CPU Spike + +# OS-49 Phase 2 / PR 1 — non-blocking spike. Runs `cargo check` on the +# nv-gha-runners shared CPU pool (`linux-{amd64,arm64}-cpu8`) with a +# GHA-backed sccache. +# +# Plan, decision thresholds, and results live in the Linear doc attached +# to OS-126 ("OS-126 — Shared CPU spike plan & results"). Dispatch this +# workflow 4–5 times after merge and record numbers there. + +on: + workflow_dispatch: + +permissions: + contents: read + packages: read + +env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: "0" + MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Wire sccache (already RUSTC_WRAPPER in mise.toml) to the GHA cache + # backend instead of the EKS memcached used by ARC. + SCCACHE_GHA_ENABLED: "true" + +jobs: + rust-check: + name: cargo check (${{ matrix.runner }}) + strategy: + fail-fast: false + matrix: + runner: [linux-amd64-cpu8, linux-arm64-cpu8] + runs-on: ${{ matrix.runner }} + container: + image: ghcr.io/nvidia/openshell/ci:latest + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + + - name: Install tools + run: mise install + + - name: Cache Rust target and registry + uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 + with: + shared-key: shadow-shared-cpu-spike-${{ matrix.runner }} + cache-directories: .cache/sccache + + - name: cargo check + run: mise x -- cargo check --workspace --all-targets + + - name: sccache stats + if: always() + run: mise x -- sccache --show-stats From 75b880b625b5efc510b106707c9f385acf38ec26 Mon Sep 17 00:00:00 2001 From: jtoelke2 <149006449+jtoelke2@users.noreply.github.com> Date: Thu, 23 Apr 2026 16:52:47 -0500 Subject: [PATCH 041/142] chore(ci): add ARC baseline collector for OS-49 runner migration (#927) Signed-off-by: Jonas Toelke --- scripts/baseline_workflow_metrics.py | 406 +++++++++++++++++++++++++++ 1 file changed, 406 insertions(+) create mode 100755 scripts/baseline_workflow_metrics.py diff --git a/scripts/baseline_workflow_metrics.py b/scripts/baseline_workflow_metrics.py new file mode 100755 index 0000000000..b937db0d8f --- /dev/null +++ b/scripts/baseline_workflow_metrics.py @@ -0,0 +1,406 @@ +#!/usr/bin/env python3 + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Record the ARC baseline for OS-49 Phase 1. + +Pulls workflow-run history from the GitHub Actions API for each of the ten +workflows currently pinned to the `build-amd64` / `build-arm64` ARC scale sets +and reports wall time, queue time, and success rate over a rolling window. +Output is both machine-readable JSON and a Markdown table so Phase 6/7 cut-over +PRs can compare like-for-like. + +Usage: + uv run python scripts/baseline_workflow_metrics.py + uv run python scripts/baseline_workflow_metrics.py --days 30 --out architecture/plans/OS-49-baseline.json + +Auth: + Relies on `gh auth login` — the script shells out to `gh api` so no token + needs to live in this process. +""" + +from __future__ import annotations + +import argparse +import dataclasses +import datetime as dt +import json +import math +import pathlib +import statistics +import subprocess +import sys +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterable + +REPO = "NVIDIA/OpenShell" + +WORKFLOWS: list[str] = [ + "branch-checks.yml", + "branch-e2e.yml", + "ci-image.yml", + "docker-build.yml", + "e2e-test.yml", + "release-canary.yml", + "release-dev.yml", + "release-tag.yml", + "release-vm-dev.yml", + "release-vm-kernel.yml", +] + +# Reusable workflows (workflow_call targets). The Actions API returns no runs +# when querying these by workflow id — their runs are rolled into the caller's +# workflow_run. For these, we scan all repo runs in the window and attribute +# via `referenced_workflows`. +REUSABLE_WORKFLOWS: set[str] = { + "docker-build.yml", + "e2e-test.yml", +} + +# Conclusions that represent a real execution on a runner. Percentile math +# excludes the rest (skipped runs in particular produce near-zero wall times +# that poison p50/p95). +RUN_TIME_CONCLUSIONS: set[str] = {"success", "failure"} + + +@dataclasses.dataclass +class RunSummary: + id: int + created_at: dt.datetime + run_started_at: dt.datetime | None + updated_at: dt.datetime + conclusion: str | None + event: str + + @property + def queue_seconds(self) -> float | None: + if self.run_started_at is None: + return None + return max(0.0, (self.run_started_at - self.created_at).total_seconds()) + + @property + def wall_seconds(self) -> float | None: + start = self.run_started_at or self.created_at + return max(0.0, (self.updated_at - start).total_seconds()) + + +@dataclasses.dataclass +class WorkflowStats: + workflow: str + window_days: int + run_count: int + success_count: int + failure_count: int + cancelled_count: int + other_count: int + wall_p50: float | None + wall_p95: float | None + wall_mean: float | None + queue_p50: float | None + queue_p95: float | None + queue_mean: float | None + reusable: bool = False + + @property + def completed(self) -> int: + return self.success_count + self.failure_count + self.cancelled_count + + @property + def success_rate(self) -> float | None: + denom = self.success_count + self.failure_count + if denom == 0: + return None + return self.success_count / denom + + +def gh_api(path: str) -> dict | list: + """Call the GitHub REST API via the gh CLI and return parsed JSON.""" + cmd = ["gh", "api", "-H", "Accept: application/vnd.github+json", path] + try: + result = subprocess.run(cmd, check=True, capture_output=True, text=True) + except FileNotFoundError: + sys.exit("gh CLI not found on PATH. Install: https://cli.github.com/") + except subprocess.CalledProcessError as exc: + sys.exit(f"gh api failed for {path}: {exc.stderr.strip()}") + return json.loads(result.stdout) + + +def parse_iso(value: str | None) -> dt.datetime | None: + if value is None: + return None + return dt.datetime.fromisoformat(value.replace("Z", "+00:00")) + + +def _run_from_raw(raw: dict) -> RunSummary | None: + created = parse_iso(raw.get("created_at")) + if created is None: + return None + return RunSummary( + id=raw["id"], + created_at=created, + run_started_at=parse_iso(raw.get("run_started_at")), + updated_at=parse_iso(raw.get("updated_at")) or created, + conclusion=raw.get("conclusion"), + event=raw.get("event", ""), + ) + + +def fetch_runs(workflow: str, since: dt.datetime) -> list[RunSummary]: + """Fetch runs for a workflow in the window via `/actions/workflows/{id}/runs`. + + Returns [] for reusable (workflow_call) workflows — the API only surfaces + them under the caller. Use `fetch_reusable_runs` for those. + """ + runs: list[RunSummary] = [] + page = 1 + cutoff = since.date().isoformat() + while True: + path = ( + f"/repos/{REPO}/actions/workflows/{workflow}/runs" + f"?created=%3E%3D{cutoff}&per_page=100&page={page}" + ) + payload = gh_api(path) + assert isinstance(payload, dict), f"unexpected payload: {type(payload)}" + batch = payload.get("workflow_runs") or [] + if not batch: + break + for raw in batch: + run = _run_from_raw(raw) + if run is None or run.created_at < since: + continue + runs.append(run) + if len(batch) < 100: + break + page += 1 + if page > 50: # safety valve: 5000 runs is plenty for 30 days + break + return runs + + +def fetch_all_repo_runs(since: dt.datetime) -> list[dict]: + """Single pass over `/actions/runs` for the window, returning raw payloads. + + Raw so callers can inspect `referenced_workflows` to attribute reusable + workflows. This is called once and shared across all reusable workflows. + """ + raws: list[dict] = [] + page = 1 + cutoff = since.date().isoformat() + while True: + path = ( + f"/repos/{REPO}/actions/runs" + f"?created=%3E%3D{cutoff}&per_page=100&page={page}" + ) + payload = gh_api(path) + assert isinstance(payload, dict), f"unexpected payload: {type(payload)}" + batch = payload.get("workflow_runs") or [] + if not batch: + break + for raw in batch: + created = parse_iso(raw.get("created_at")) + if created is None or created < since: + continue + raws.append(raw) + if len(batch) < 100: + break + page += 1 + if page > 200: # safety valve: 20k runs — generous for 30 days + break + return raws + + +def fetch_reusable_runs(workflow: str, all_repo_runs: list[dict]) -> list[RunSummary]: + """Attribute a reusable workflow via `referenced_workflows` on caller runs. + + Wall/queue times come from the caller run, so they are caller-inclusive — + a caller that inlines other jobs alongside the reusable workflow will + overstate the reusable piece. Annotated in the rendered output. + """ + suffix = f".github/workflows/{workflow}" + runs: list[RunSummary] = [] + for raw in all_repo_runs: + refs = raw.get("referenced_workflows") or [] + # GitHub returns paths like "org/repo/.github/workflows/foo.yml@" + # — strip the `@ref` suffix before matching. + if not any(r.get("path", "").split("@", 1)[0].endswith(suffix) for r in refs): + continue + run = _run_from_raw(raw) + if run is None: + continue + runs.append(run) + return runs + + +def _percentile(values: Iterable[float], p: float) -> float | None: + sample = sorted(v for v in values if v is not None) + if not sample: + return None + if len(sample) == 1: + return sample[0] + k = (len(sample) - 1) * p + lo = math.floor(k) + hi = math.ceil(k) + if lo == hi: + return sample[int(k)] + return sample[lo] + (sample[hi] - sample[lo]) * (k - lo) + + +def summarize( + workflow: str, + window_days: int, + runs: list[RunSummary], + reusable: bool = False, +) -> WorkflowStats: + completed = [r for r in runs if r.conclusion is not None] + success_count = sum(1 for r in completed if r.conclusion == "success") + failure_count = sum(1 for r in completed if r.conclusion == "failure") + cancelled_count = sum(1 for r in completed if r.conclusion == "cancelled") + other_count = len(completed) - success_count - failure_count - cancelled_count + # p50/p95 over real executions only — skipped/cancelled/startup_failure + # produce near-zero wall times and would poison percentiles. + executed = [r for r in completed if r.conclusion in RUN_TIME_CONCLUSIONS] + wall = [r.wall_seconds for r in executed if r.wall_seconds is not None] + queue = [r.queue_seconds for r in executed if r.queue_seconds is not None] + return WorkflowStats( + workflow=workflow, + window_days=window_days, + run_count=len(runs), + success_count=success_count, + failure_count=failure_count, + cancelled_count=cancelled_count, + other_count=other_count, + wall_p50=_percentile(wall, 0.50), + wall_p95=_percentile(wall, 0.95), + wall_mean=statistics.fmean(wall) if wall else None, + queue_p50=_percentile(queue, 0.50), + queue_p95=_percentile(queue, 0.95), + queue_mean=statistics.fmean(queue) if queue else None, + reusable=reusable, + ) + + +def fmt_seconds(value: float | None) -> str: + if value is None: + return "—" + if value < 60: + return f"{value:.0f}s" + if value < 3600: + return f"{value / 60:.1f}m" + return f"{value / 3600:.2f}h" + + +def fmt_rate(value: float | None) -> str: + if value is None: + return "—" + return f"{value * 100:.0f}%" + + +def render_markdown(stats: list[WorkflowStats], since: dt.datetime) -> str: + lines: list[str] = [] + lines.append( + f"# OS-49 Phase 1 — ARC baseline ({REPO}, last {stats[0].window_days} days)" + ) + lines.append("") + lines.append(f"Window: `{since.date().isoformat()}` → today (UTC).") + lines.append("") + lines.append( + "| Workflow | Runs | Success | Wall p50 | Wall p95 | Queue p50 | Queue p95 |" + ) + lines.append("|---|---:|---:|---:|---:|---:|---:|") + has_reusable = False + for s in stats: + name = f"`{s.workflow}`" + if s.reusable: + name += " †" + has_reusable = True + lines.append( + f"| {name} | {s.run_count} | {fmt_rate(s.success_rate)} " + f"| {fmt_seconds(s.wall_p50)} | {fmt_seconds(s.wall_p95)} " + f"| {fmt_seconds(s.queue_p50)} | {fmt_seconds(s.queue_p95)} |" + ) + lines.append("") + lines.append( + "Percentiles cover runs with conclusion `success` or `failure` only — " + "`skipped`/`cancelled`/`startup_failure` are excluded so `if:` guards and early aborts " + "don't poison wall-time p50. Success rate = `success / (success + failure)`. " + "Queue time = `run_started_at − created_at`. Wall time = `updated_at − run_started_at`." # noqa: RUF001 — U+2212 minus rendered in output markdown + ) + if has_reusable: + lines.append("") + lines.append( + "† Reusable workflow (`workflow_call`). Runs attributed via " + "`referenced_workflows` on caller runs; wall/queue times are the " + "caller's totals, so they overstate the reusable piece." + ) + return "\n".join(lines) + "\n" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=(__doc__ or "").splitlines()[0]) + parser.add_argument( + "--days", + type=int, + default=30, + help="Look-back window in days (default: 30)", + ) + parser.add_argument( + "--out", + type=pathlib.Path, + default=pathlib.Path("architecture/plans/OS-49-baseline.json"), + help="Where to write the JSON report (default: architecture/plans/OS-49-baseline.json)", + ) + parser.add_argument( + "--md", + type=pathlib.Path, + default=pathlib.Path("architecture/plans/OS-49-baseline.md"), + help="Where to write the Markdown report (default: architecture/plans/OS-49-baseline.md)", + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + since = dt.datetime.now(dt.UTC) - dt.timedelta(days=args.days) + all_stats: list[WorkflowStats] = [] + # Only do the expensive all-repo scan if at least one reusable workflow + # appears in the target list. + need_repo_scan = any(w in REUSABLE_WORKFLOWS for w in WORKFLOWS) + all_repo_runs: list[dict] = [] + if need_repo_scan: + print("• (scanning all repo runs for reusable attribution)", file=sys.stderr) + all_repo_runs = fetch_all_repo_runs(since) + for workflow in WORKFLOWS: + print(f"• {workflow}", file=sys.stderr) + if workflow in REUSABLE_WORKFLOWS: + runs = fetch_reusable_runs(workflow, all_repo_runs) + all_stats.append(summarize(workflow, args.days, runs, reusable=True)) + else: + runs = fetch_runs(workflow, since) + all_stats.append(summarize(workflow, args.days, runs)) + + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text( + json.dumps( + { + "repo": REPO, + "window_days": args.days, + "generated_at": dt.datetime.now(dt.UTC).isoformat(), + "workflows": [dataclasses.asdict(s) for s in all_stats], + }, + indent=2, + default=str, + ) + + "\n", + encoding="utf-8", + ) + args.md.write_text(render_markdown(all_stats, since), encoding="utf-8") + print(f"wrote {args.out}", file=sys.stderr) + print(f"wrote {args.md}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From ef2d993896ed50e045704f86c67ad1b22ff08218 Mon Sep 17 00:00:00 2001 From: jtoelke2 <149006449+jtoelke2@users.noreply.github.com> Date: Thu, 23 Apr 2026 22:58:21 -0500 Subject: [PATCH 042/142] fix(ci): expose GHA sccache env in shadow-shared-cpu-spike (#950) Signed-off-by: Jonas Toelke --- .github/workflows/shadow-shared-cpu-spike.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/shadow-shared-cpu-spike.yml b/.github/workflows/shadow-shared-cpu-spike.yml index 6dae8ad9d0..f8d1785a08 100644 --- a/.github/workflows/shadow-shared-cpu-spike.yml +++ b/.github/workflows/shadow-shared-cpu-spike.yml @@ -43,6 +43,14 @@ jobs: - name: Install tools run: mise install + - name: Configure GHA sccache backend + # Exposes ACTIONS_CACHE_URL / ACTIONS_RUNTIME_TOKEN to subsequent steps + # so sccache (wrapped around rustc via RUSTC_WRAPPER in mise.toml) can + # initialize the GHA cache. Without this, sccache fails at startup with + # "cache url for ghac not found". The action also installs its own + # sccache binary; harmless since mise's sccache remains on PATH. + uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9 + - name: Cache Rust target and registry uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 with: From a4dfa5ad789c840498023c28795aedb133a42ded Mon Sep 17 00:00:00 2001 From: jtoelke2 <149006449+jtoelke2@users.noreply.github.com> Date: Thu, 23 Apr 2026 22:58:31 -0500 Subject: [PATCH 043/142] feat(ci): add driver input to setup-buildx action (#941) Signed-off-by: Jonas Toelke --- .github/actions/setup-buildx/action.yml | 28 +++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/.github/actions/setup-buildx/action.yml b/.github/actions/setup-buildx/action.yml index f7c8dbbd01..bd178e1732 100644 --- a/.github/actions/setup-buildx/action.yml +++ b/.github/actions/setup-buildx/action.yml @@ -1,15 +1,22 @@ name: Setup Docker Buildx description: > - Create a multi-arch Docker Buildx builder using remote BuildKit nodes. - The builder is automatically removed when the job finishes (cleanup is - enabled by default in docker/setup-buildx-action). + Create a Docker Buildx builder. Two modes: + * driver=remote (default) — multi-arch builder against in-cluster BuildKit + pods. Requires EKS connectivity. Behaviour unchanged from prior versions. + * driver=local — single-node buildx on the local docker-container driver. + Pair with cache-to/cache-from=type=gha on build steps for persistence. + Works on nv-gha-runners; no EKS needed. + Cleanup is automatic when the job finishes (docker/setup-buildx-action default). inputs: + driver: + description: "buildx driver: 'remote' or 'local'" + default: remote amd64-endpoint: - description: BuildKit endpoint for linux/amd64 + description: BuildKit endpoint for linux/amd64 (remote driver only) default: tcp://buildkit-amd64.buildkit:1234 arm64-endpoint: - description: BuildKit endpoint for linux/arm64 + description: BuildKit endpoint for linux/arm64 (remote driver only) default: tcp://buildkit-arm64.buildkit:1234 name: description: Builder instance name @@ -18,7 +25,8 @@ inputs: runs: using: composite steps: - - name: Set up Docker Buildx + - name: Set up Docker Buildx (remote) + if: inputs.driver == 'remote' uses: docker/setup-buildx-action@v3 with: name: ${{ inputs.name }} @@ -28,3 +36,11 @@ runs: append: | - endpoint: ${{ inputs.arm64-endpoint }} platforms: linux/arm64 + + - name: Set up Docker Buildx (local) + if: inputs.driver == 'local' + uses: docker/setup-buildx-action@v3 + with: + name: ${{ inputs.name }} + driver: docker-container + platforms: linux/amd64,linux/arm64 From 0d301d578f145ef24d26ab27352b28a0e7812c0d Mon Sep 17 00:00:00 2001 From: mjamiv <142179942+mjamiv@users.noreply.github.com> Date: Thu, 23 Apr 2026 23:49:15 -0600 Subject: [PATCH 044/142] fix(cli): preserve directory basename when uploading to sandbox (#952) Previously `openshell sandbox upload ` flattened the local directory's contents into the remote destination, dropping the source directory's basename. This diverged from the standard `scp -r` / `cp -r` semantics that operators expect from any *nix-style transfer tool, and could silently shadow files at the destination. Now an upload of `foo` to `/sandbox/dest/` lands at `/sandbox/dest/foo/`, matching `scp -r foo remote:/sandbox/dest/` and `cp -r foo /sandbox/dest/`. Paths without a meaningful basename (`.`, `/`) preserve the legacy flat extraction. Closes #885 --- crates/openshell-cli/src/ssh.rs | 52 +++++++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/crates/openshell-cli/src/ssh.rs b/crates/openshell-cli/src/ssh.rs index 0f7e8ee0b0..54321bfccd 100644 --- a/crates/openshell-cli/src/ssh.rs +++ b/crates/openshell-cli/src/ssh.rs @@ -527,7 +527,9 @@ async fn ssh_tar_upload( .append_path_with_name(&local_path, &tar_name) .into_diagnostic()?; } else if local_path.is_dir() { - archive.append_dir_all(".", &local_path).into_diagnostic()?; + archive + .append_dir_all(&tar_name, &local_path) + .into_diagnostic()?; } else { return Err(miette::miette!( "local path does not exist: {}", @@ -665,8 +667,11 @@ pub async fn sandbox_sync_up( .ok_or_else(|| miette::miette!("path has no file name"))? .to_os_string() } else { - // For directories the tar_name is unused — append_dir_all uses "." - ".".into() + // For directories, wrap contents under the source basename so uploads + // land at `//...` — matches `scp -r` and `cp -r`. Falls + // back to "." for paths with no meaningful basename (`.`, `/`), which + // preserves the legacy flatten behavior in those edge cases. + directory_upload_prefix(local_path) }; ssh_tar_upload( @@ -682,6 +687,19 @@ pub async fn sandbox_sync_up( .await } +/// Compute the tar entry prefix for a directory upload. +/// +/// Returns the directory's basename for any path with a meaningful basename; +/// callers extracting at `` will see contents wrapped under +/// `//...`. Returns `"."` for paths without a basename +/// (e.g. `.` or `/`), which produces flat extraction at ``. +fn directory_upload_prefix(local_path: &Path) -> std::ffi::OsString { + local_path + .file_name() + .map(|n| n.to_os_string()) + .unwrap_or_else(|| ".".into()) +} + /// Pull a path from a sandbox to a local destination using tar-over-SSH. pub async fn sandbox_sync_down( server: &str, @@ -1280,6 +1298,34 @@ mod tests { assert_eq!(split_sandbox_path("/a/b/c/d.txt"), ("/a/b/c", "d.txt")); } + #[test] + fn directory_upload_prefix_uses_basename_for_named_directories() { + assert_eq!( + directory_upload_prefix(Path::new("/tmp/upload-test")), + std::ffi::OsString::from("upload-test") + ); + assert_eq!( + directory_upload_prefix(Path::new("foo")), + std::ffi::OsString::from("foo") + ); + assert_eq!( + directory_upload_prefix(Path::new("./parent/nested")), + std::ffi::OsString::from("nested") + ); + } + + #[test] + fn directory_upload_prefix_falls_back_to_dot_for_basename_less_paths() { + assert_eq!( + directory_upload_prefix(Path::new(".")), + std::ffi::OsString::from(".") + ); + assert_eq!( + directory_upload_prefix(Path::new("/")), + std::ffi::OsString::from(".") + ); + } + #[test] fn split_sandbox_path_handles_root_and_bare_names() { // File directly under root From 7f8e2109e3ba1bd453e755b8d97642a9130f5c55 Mon Sep 17 00:00:00 2001 From: "John T. Myers" <9696606+johntmyers@users.noreply.github.com> Date: Fri, 24 Apr 2026 07:49:49 -0700 Subject: [PATCH 045/142] fix(sandbox): route console logs to stderr (#949) Signed-off-by: John Myers Co-authored-by: John Myers --- architecture/sandbox-custom-containers.md | 2 +- crates/openshell-sandbox/src/main.rs | 14 +++---- .../openshell-sandbox/tests/stdout_logging.rs | 41 +++++++++++++++++++ 3 files changed, 49 insertions(+), 8 deletions(-) create mode 100644 crates/openshell-sandbox/tests/stdout_logging.rs diff --git a/architecture/sandbox-custom-containers.md b/architecture/sandbox-custom-containers.md index cbdc9a6661..5d482ffe04 100644 --- a/architecture/sandbox-custom-containers.md +++ b/architecture/sandbox-custom-containers.md @@ -96,7 +96,7 @@ openshell sandbox create --from ./my-sandbox/ # directory with Dockerfile The `openshell-sandbox` supervisor adapts to arbitrary environments: -- **Log file fallback**: Attempts to open `/var/log/openshell.log` for append; silently falls back to stdout-only logging if the path is not writable. +- **Log file fallback**: Attempts to open `/var/log/openshell.log` for append; if the path is not writable, the supervisor keeps console shorthand logging on stderr only. - **Command resolution**: Executes the command from CLI args, then the `OPENSHELL_SANDBOX_COMMAND` env var (set to `sleep infinity` by the server), then `/bin/bash` as a last resort. - **Startup seccomp prelude**: Before parsing CLI args or starting the async runtime, the supervisor sets `PR_SET_NO_NEW_PRIVS` and installs a narrow seccomp filter that blocks mount/remount, the new mount API syscalls, module loading, kexec, `bpf`, `perf_event_open`, and `userfaultfd`. This closes the privileged remount window while still leaving required child-setup syscalls such as `setns` available. - **Network namespace**: Requires successful namespace creation for proxy isolation; startup fails in proxy mode if required capabilities (`CAP_NET_ADMIN`, `CAP_SYS_ADMIN`) or `iproute2` are unavailable. If the `iptables` package is present, the supervisor installs OUTPUT chain rules (LOG + REJECT) inside the namespace to provide fast-fail behavior (immediate `ECONNREFUSED` instead of a 30-second timeout) and diagnostic logging when processes attempt direct connections that bypass the HTTP CONNECT proxy. If `iptables` is absent, the supervisor logs a warning and continues — core network isolation still works via routing. diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index b6b8ac24a1..c7aa41a8f1 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -101,7 +101,7 @@ struct Args { fn main() -> Result<()> { let args = Args::parse(); - // Try to open a rolling log file; fall back to stdout-only logging if it fails + // Try to open a rolling log file; fall back to stderr-only logging if it fails // (e.g., /var/log is not writable in custom workload images). // Rotates daily, keeps the 3 most recent files to bound disk usage. let file_logging = tracing_appender::rolling::RollingFileAppender::builder() @@ -116,7 +116,7 @@ fn main() -> Result<()> { (writer, guard) }); - let stdout_filter = + let console_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)); let runtime = tokio::runtime::Builder::new_multi_thread() @@ -176,9 +176,9 @@ fn main() -> Result<()> { tracing_subscriber::registry() .with( - OcsfShorthandLayer::new(std::io::stdout()) + OcsfShorthandLayer::new(std::io::stderr()) .with_non_ocsf(true) - .with_filter(stdout_filter), + .with_filter(console_filter), ) .with( OcsfShorthandLayer::new(file_writer) @@ -192,14 +192,14 @@ fn main() -> Result<()> { } else { tracing_subscriber::registry() .with( - OcsfShorthandLayer::new(std::io::stdout()) + OcsfShorthandLayer::new(std::io::stderr()) .with_non_ocsf(true) - .with_filter(stdout_filter), + .with_filter(console_filter), ) .with(push_layer) .init(); // Log the warning after the subscriber is initialized - warn!("Could not open /var/log for log rotation; using stdout-only logging"); + warn!("Could not open /var/log for log rotation; using stderr-only logging"); (None, None) }; diff --git a/crates/openshell-sandbox/tests/stdout_logging.rs b/crates/openshell-sandbox/tests/stdout_logging.rs new file mode 100644 index 0000000000..c4f5213de8 --- /dev/null +++ b/crates/openshell-sandbox/tests/stdout_logging.rs @@ -0,0 +1,41 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::process::Command; + +#[test] +fn startup_logs_go_to_stderr_not_stdout() { + let output = Command::new(env!("CARGO_BIN_EXE_openshell-sandbox")) + .arg("--") + .arg("/usr/bin/printf") + .arg("hello") + .env("OPENSHELL_LOG_LEVEL", "info") + .env_remove("RUST_LOG") + .env_remove("OPENSHELL_POLICY_RULES") + .env_remove("OPENSHELL_POLICY_DATA") + .env_remove("OPENSHELL_SANDBOX_ID") + .env_remove("OPENSHELL_ENDPOINT") + .output() + .expect("spawn openshell-sandbox"); + + assert!( + !output.status.success(), + "expected sandbox startup to fail without a policy source" + ); + + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + + assert!( + stdout.trim().is_empty(), + "expected startup logs on stderr only, got stdout: {stdout}" + ); + assert!( + stderr.contains("Starting sandbox"), + "expected startup log on stderr, got: {stderr}" + ); + assert!( + stderr.contains("Sandbox policy required"), + "expected missing-policy error on stderr, got: {stderr}" + ); +} From 87f50f5e5d1d22d55a31a1b7ed225a553dde8e82 Mon Sep 17 00:00:00 2001 From: Derek Carr Date: Fri, 24 Apr 2026 10:51:04 -0400 Subject: [PATCH 046/142] fix(e2e): add /dev/urandom to provider test sandbox policy (#948) Python runtime requires /dev/urandom access during initialization to seed the hash randomizer. The _default_policy() in provider tests was missing this path, causing exec_python tests to fail with: 'Fatal Python error: _Py_HashRandomization_Init: failed to get random numbers to initialize Python'. Add /dev/urandom to read_only paths to match the policy used in test_sandbox_policy.py, allowing Python to initialize successfully. Signed-off-by: Derek Carr --- e2e/python/test_sandbox_providers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/python/test_sandbox_providers.py b/e2e/python/test_sandbox_providers.py index 4cbd76b501..16dfb32372 100644 --- a/e2e/python/test_sandbox_providers.py +++ b/e2e/python/test_sandbox_providers.py @@ -36,7 +36,7 @@ def _default_policy() -> sandbox_pb2.SandboxPolicy: version=1, filesystem=sandbox_pb2.FilesystemPolicy( include_workdir=True, - read_only=["/usr", "/lib", "/etc", "/app"], + read_only=["/usr", "/lib", "/etc", "/app", "/dev/urandom"], read_write=["/sandbox", "/tmp"], ), landlock=sandbox_pb2.LandlockPolicy(compatibility="best_effort"), From a34b25a5ba080485c0f65254a3bd8896b8fb648b Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Fri, 24 Apr 2026 08:28:40 -0700 Subject: [PATCH 047/142] test(e2e): fix rust upload path assertions (#960) Signed-off-by: Drew Newberry --- e2e/rust/tests/sync.rs | 6 +++--- e2e/rust/tests/upload_create.rs | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/e2e/rust/tests/sync.rs b/e2e/rust/tests/sync.rs index 9761390356..61dac81d44 100644 --- a/e2e/rust/tests/sync.rs +++ b/e2e/rust/tests/sync.rs @@ -58,7 +58,7 @@ async fn sandbox_file_upload_download_round_trip() { let download_str = download_dir.to_str().expect("download path is UTF-8"); guard - .download("/sandbox/uploaded", download_str) + .download("/sandbox/uploaded/upload", download_str) .await .expect("download directory"); @@ -108,7 +108,7 @@ async fn sandbox_file_upload_download_round_trip() { let large_down_str = large_down.to_str().expect("large_down path is UTF-8"); guard - .download("/sandbox/large_test", large_down_str) + .download("/sandbox/large_test/large_upload", large_down_str) .await .expect("download large file"); @@ -257,7 +257,7 @@ async fn upload_respects_gitignore_by_default() { let download_str = download_dir.to_str().expect("verify path is UTF-8"); guard - .download("/sandbox/filtered", download_str) + .download("/sandbox/filtered/repo", download_str) .await .expect("download filtered upload"); diff --git a/e2e/rust/tests/upload_create.rs b/e2e/rust/tests/upload_create.rs index 86721db51b..c7538e5f3e 100644 --- a/e2e/rust/tests/upload_create.rs +++ b/e2e/rust/tests/upload_create.rs @@ -31,13 +31,14 @@ async fn create_with_upload_provides_files_to_command() { fs::write(upload_dir.join("src/main.py"), "print('hello')").expect("write main.py"); let upload_str = upload_dir.to_str().expect("upload path is UTF-8"); + let remote_marker = "/sandbox/data/project/marker.txt"; // The command reads the marker file — if upload worked, its content // appears in the output. let mut guard = SandboxGuard::create_with_upload( upload_str, "/sandbox/data", - &["cat", "/sandbox/data/marker.txt"], + &["cat", remote_marker], ) .await .expect("sandbox create --upload"); From 8cf5ebdc8966bb42837aa8b57a5eee442d8f5e05 Mon Sep 17 00:00:00 2001 From: "John T. Myers" <9696606+johntmyers@users.noreply.github.com> Date: Fri, 24 Apr 2026 09:26:50 -0700 Subject: [PATCH 048/142] test(e2e): fix gitignore upload assertion path (#962) Signed-off-by: John Myers Co-authored-by: John Myers --- e2e/rust/tests/sync.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/rust/tests/sync.rs b/e2e/rust/tests/sync.rs index 61dac81d44..29ffdc00d4 100644 --- a/e2e/rust/tests/sync.rs +++ b/e2e/rust/tests/sync.rs @@ -257,7 +257,7 @@ async fn upload_respects_gitignore_by_default() { let download_str = download_dir.to_str().expect("verify path is UTF-8"); guard - .download("/sandbox/filtered/repo", download_str) + .download("/sandbox/filtered", download_str) .await .expect("download filtered upload"); From 77a88c313b413a3ed55b216ef71f4cc9a7b50964 Mon Sep 17 00:00:00 2001 From: jtoelke2 <149006449+jtoelke2@users.noreply.github.com> Date: Fri, 24 Apr 2026 11:39:09 -0500 Subject: [PATCH 049/142] fix(ci): partition GHA sccache cache per arch in shadow spike (#961) Without a per-arch SCCACHE_GHA_VERSION, both matrix jobs (linux-amd64-cpu8 and linux-arm64-cpu8) target the same GHA cache key. Run 1 on 2026-04-24 showed this concretely: arm64 started first and landed 575/1064 cache writes; amd64 started ~1 min later and every one of its 1062 writes failed, leaving the cache with no amd64 entries at all. Partitioning by matrix.runner ensures each arch gets its own cache namespace, letting us see real warm-cache behavior on the subsequent Run 2-4 dispatches. Signed-off-by: Jonas Toelke --- .github/workflows/shadow-shared-cpu-spike.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/shadow-shared-cpu-spike.yml b/.github/workflows/shadow-shared-cpu-spike.yml index f8d1785a08..c264e595f7 100644 --- a/.github/workflows/shadow-shared-cpu-spike.yml +++ b/.github/workflows/shadow-shared-cpu-spike.yml @@ -31,6 +31,14 @@ jobs: matrix: runner: [linux-amd64-cpu8, linux-arm64-cpu8] runs-on: ${{ matrix.runner }} + env: + # Partition the GHA sccache cache per-arch. Without this, matrix jobs + # collide on the same cache key, and the later-starting job's writes + # fail with 409 Conflict while the earlier one's writes land — leaving + # subsequent runs with asymmetric and mostly-empty caches. + # (Run 1 on 2026-04-24 showed amd64 with 0/1062 writes succeeding + # while arm64 got 575/1064.) + SCCACHE_GHA_VERSION: ${{ matrix.runner }} container: image: ghcr.io/nvidia/openshell/ci:latest credentials: From d44d8a1e2716de855bd299079d1eb5a5b476874b Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Fri, 24 Apr 2026 12:30:14 -0500 Subject: [PATCH 050/142] feat: Openshell driver podman (#904) * feat(podman): add Podman compute driver for rootless sandbox management Adds openshell-driver-podman, a new compute driver that manages OpenShell sandboxes as rootless Podman containers via the Podman REST API over a Unix socket. Enables local workstation sandboxes without Kubernetes. Driver features: - Bridge networking with ephemeral host-port mapping for rootless SSH reachability - Named volumes for workspace storage, Podman native health checks, GPU via CDI - Supervisor binary sideloaded via image volume mount (BYOC-compatible) - SSH handshake secret injected via Podman secrets API (not plaintext env) - Typed ContainerSpec structs, input validation, and path-traversal guards - Cgroups v2 required; fails fast on v1 hosts - Bounded event stream buffer; watch stream reconnection handled by server watch_loop - Graceful shutdown and standalone driver binary with gRPC bridge Rootless-specific fixes: - Skip drop_privileges when user namespace lacks SETUID/SETGID/DAC_READ_SEARCH caps - Add /run/netns tmpfs mount for ip netns in rootless containers - Use secret_env map (not secrets array) for env-var injection in libpod API - Resolve SSH endpoint to 127.0.0.1: instead of unreachable bridge IP Server/sandbox hardening: - Split loopback and link-local SSRF gates; Podman/VM drivers allow loopback - Close SSRF bypass in SSH tunnel Host path by resolving DNS before connecting - Prevent OPENSHELL_* env var override by user-supplied spec environment maps - Disable SQLite pool idle_timeout/max_lifetime for in-memory databases - Emit deleted_event on 404-during-inspect instead of regressing sandbox phase - Key delete cleanup by stable sandbox_id to survive container label drift CLI fixes: - Restore --name as a named flag on sandbox create (not positional) - Fix exec command arg parsing to not consume sandboxed-command flags - Propagate SSH verbosity via OPENSHELL_SSH_LOG_LEVEL Build tooling: - Add tasks/scripts/container-engine.sh: auto-detects Podman or Docker, exposes unified ce_* helpers; all build/cluster/VM scripts updated to use it - Add docker:build:supervisor mise task for standalone supervisor image - Add openshell-driver-podman to Dockerfile.images pre-fetch/build stages - Add e2e/rust/e2e-podman.sh and e2e:podman mise task for full lifecycle testing Signed-off-by: Adam Miller * fix(driver-podman): derive grpc endpoint from server bind port When a user starts the gateway on a non-default port (e.g. --port 8081), sandbox containers were receiving OPENSHELL_ENDPOINT pointing at the default port 8080. The driver's auto-detection fallback read OPENSHELL_BIND_ADDRESS from the environment, which was stale or unset, and fell back to DEFAULT_SERVER_PORT. Add gateway_port to PodmanComputeConfig and thread config.bind_address.port() from the server into the driver so the fallback uses the actual listening port. Remove the OPENSHELL_BIND_ADDRESS env var read and the extract_port_from_bind_address helper which are no longer needed. Add --gateway-port / OPENSHELL_GATEWAY_PORT to the standalone driver binary for parity when the driver is run outside the embedded server path. Signed-off-by: Adam Miller * fix(driver-podman): address PR feedback on env test safety and cluster DNS docs Replace hand-rolled unsafe TempEnvVar RAII guard with temp_env::with_vars and a static ENV_LOCK mutex, fixing a data race in parallel test execution. The prior safety comment incorrectly claimed Cargo runs tests single-threaded. Update debug-openshell-cluster skill to accurately document the DNS proxy strategy (setup_dns_proxy + public DNS fallback) and clarify the separation between cluster DNS and sandbox agent DNS enforcement. Signed-off-by: Adam Miller * fix(e2e): resolve CI failures in auth timeout, test harness, and formatting - Short-circuit browser_auth_flow when OPENSHELL_NO_BROWSER=1 instead of waiting the full 120s AUTH_TIMEOUT for a callback that never arrives - Add timeout to SandboxGuard::create() and create_with_upload() to prevent indefinite hangs (matches create_keep() which already had one) - Add missing '--' separator in no_proxy test before command args - Add #![cfg(feature = "e2e")] gate to sandbox_lifecycle.rs - Run cargo fmt on openshell-driver-podman - Refine cluster DNS docs for Podman in debug-openshell-cluster skill Signed-off-by: Adam Miller * refactor(server): remove allows_loopback_endpoints from ComputeRuntime SSRF protection is now handled at the network and proxy layers (openshell-core net.rs, openshell-sandbox proxy.rs) rather than requiring per-driver flags on ComputeRuntime. Update architecture docs to reflect supervisor relay SSH transport and add rootless networking deep-dive. Signed-off-by: Adam Miller --------- Signed-off-by: Adam Miller --- .../skills/debug-openshell-cluster/SKILL.md | 45 +- Cargo.lock | 211 +++-- architecture/podman-driver.md | 259 ++++++ architecture/podman-rootless-networking.md | 372 ++++++++ architecture/sandbox-connect.md | 10 +- crates/openshell-cli/src/auth.rs | 45 +- crates/openshell-cli/src/main.rs | 73 +- crates/openshell-cli/src/ssh.rs | 7 +- crates/openshell-core/src/config.rs | 61 +- crates/openshell-core/src/error.rs | 17 + crates/openshell-core/src/lib.rs | 2 +- crates/openshell-driver-podman/Cargo.toml | 40 + crates/openshell-driver-podman/src/client.rs | 829 +++++++++++++++++ crates/openshell-driver-podman/src/config.rs | 199 +++++ .../openshell-driver-podman/src/container.rs | 830 ++++++++++++++++++ crates/openshell-driver-podman/src/driver.rs | 822 +++++++++++++++++ crates/openshell-driver-podman/src/grpc.rs | 412 +++++++++ crates/openshell-driver-podman/src/lib.rs | 13 + crates/openshell-driver-podman/src/main.rs | 131 +++ crates/openshell-driver-podman/src/watcher.rs | 550 ++++++++++++ .../src/sandbox/linux/netns.rs | 44 +- crates/openshell-sandbox/src/ssh.rs | 69 ++ crates/openshell-server/Cargo.toml | 1 + crates/openshell-server/src/cli.rs | 13 +- crates/openshell-server/src/compute/mod.rs | 52 +- crates/openshell-server/src/lib.rs | 78 +- .../src/persistence/sqlite.rs | 22 +- crates/openshell-vm/scripts/build-rootfs.sh | 27 +- deploy/docker/Dockerfile.images | 5 + e2e/rust/e2e-podman.sh | 168 ++++ e2e/rust/src/harness/sandbox.rs | 14 +- e2e/rust/tests/no_proxy.rs | 2 +- e2e/rust/tests/sandbox_lifecycle.rs | 2 + mise.toml | 3 +- scripts/docker-cleanup.sh | 46 +- scripts/remote-deploy.sh | 6 +- tasks/docker.toml | 24 +- tasks/python.toml | 4 +- tasks/scripts/cluster-bootstrap.sh | 42 +- tasks/scripts/cluster-deploy-fast.sh | 33 +- tasks/scripts/cluster-push-component.sh | 13 +- tasks/scripts/cluster.sh | 7 +- tasks/scripts/container-engine.sh | 324 +++++++ tasks/scripts/docker-build-ci.sh | 5 +- tasks/scripts/docker-build-image.sh | 163 ++-- tasks/scripts/docker-publish-multiarch.sh | 134 ++- tasks/scripts/sandbox.sh | 5 +- tasks/scripts/vm/build-rootfs-tarball.sh | 17 +- tasks/scripts/vm/sync-vm-rootfs.sh | 8 +- tasks/test.toml | 5 + 50 files changed, 5873 insertions(+), 391 deletions(-) create mode 100644 architecture/podman-driver.md create mode 100644 architecture/podman-rootless-networking.md create mode 100644 crates/openshell-driver-podman/Cargo.toml create mode 100644 crates/openshell-driver-podman/src/client.rs create mode 100644 crates/openshell-driver-podman/src/config.rs create mode 100644 crates/openshell-driver-podman/src/container.rs create mode 100644 crates/openshell-driver-podman/src/driver.rs create mode 100644 crates/openshell-driver-podman/src/grpc.rs create mode 100644 crates/openshell-driver-podman/src/lib.rs create mode 100644 crates/openshell-driver-podman/src/main.rs create mode 100644 crates/openshell-driver-podman/src/watcher.rs create mode 100755 e2e/rust/e2e-podman.sh create mode 100755 tasks/scripts/container-engine.sh diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index 3d2e6f7827..702cda1327 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -7,16 +7,16 @@ description: Debug why a openshell cluster failed to start or is unhealthy. Use Diagnose why a openshell cluster failed to start after `openshell gateway start`. -Use **only** `openshell` CLI commands (`openshell status`, `openshell doctor logs`, `openshell doctor exec`) to inspect and fix the cluster. Do **not** use raw `docker`, `ssh`, or `kubectl` commands directly — always go through the `openshell doctor` interface. The CLI auto-resolves local vs remote gateways, so the same commands work everywhere. +Use **only** `openshell` CLI commands (`openshell status`, `openshell doctor logs`, `openshell doctor exec`) to inspect and fix the cluster. Do **not** use raw `docker`, `podman`, `ssh`, or `kubectl` commands directly — always go through the `openshell doctor` interface. The CLI auto-resolves local vs remote gateways, so the same commands work everywhere. ## Overview -`openshell gateway start` creates a Docker container running k3s with the OpenShell server deployed via Helm. The deployment stages, in order, are: +`openshell gateway start` creates a container (via Docker or Podman) running k3s with the OpenShell server deployed via Helm. The build and deploy scripts use a container-engine abstraction layer (`tasks/scripts/container-engine.sh`) that auto-detects Docker or Podman and provides a unified `ce` command interface. Set `CONTAINER_ENGINE=docker` or `CONTAINER_ENGINE=podman` to override auto-detection. The deployment stages, in order, are: 1. **Pre-deploy check**: `openshell gateway start` in interactive mode prompts to **reuse** (keep volume, clean stale nodes) or **recreate** (destroy everything, fresh start). `mise run cluster` always recreates before deploy. 2. Ensure cluster image is available (local build or remote pull) -3. Create Docker network (`openshell-cluster`) and volume (`openshell-cluster-{name}`) -4. Create and start a privileged Docker container (`openshell-cluster-{name}`) +3. Create container network (`openshell-cluster`) and volume (`openshell-cluster-{name}`) +4. Create and start a privileged container (`openshell-cluster-{name}`) 5. Wait for k3s to generate kubeconfig (up to 60s) 6. **Clean stale nodes**: Remove any `NotReady` k3s nodes left over from previous container instances that reused the same persistent volume 7. **Prepare local images** (if `OPENSHELL_PUSH_IMAGES` is set): In `internal` registry mode, bootstrap waits for the in-cluster registry and pushes tagged images there. In `external` mode, bootstrap uses legacy `ctr -n k8s.io images import` push-mode behavior. @@ -28,10 +28,11 @@ Use **only** `openshell` CLI commands (`openshell status`, `openshell doctor log - TLS secrets `openshell-server-tls` and `openshell-client-tls` exist in `openshell` namespace - Sandbox supervisor binary exists at `/opt/openshell/bin/openshell-sandbox` (emits `HEALTHCHECK_MISSING_SUPERVISOR` marker if absent) -For local deploys, metadata endpoint selection now depends on Docker connectivity: +For local deploys, metadata endpoint selection depends on the container engine and its connectivity: - default local Docker socket (`unix:///var/run/docker.sock`): `https://127.0.0.1:{port}` (default port 8080) - TCP Docker daemon (`DOCKER_HOST=tcp://:`): `https://:{port}` for non-loopback hosts +- Podman (rootless): `https://127.0.0.1:{port}` via `host.containers.internal` (the Podman equivalent of `host.docker.internal`) The host port is configurable via `--port` on `openshell gateway start` (default 8080) and is stored in `ClusterMetadata.gateway_port`. @@ -41,7 +42,8 @@ The default cluster name is `openshell`. The container is `openshell-cluster-{na ## Prerequisites -- Docker must be running (locally or on the remote host) +- Docker or Podman must be running (locally or on the remote host). The build system auto-detects which engine is available; set `CONTAINER_ENGINE=docker` or `CONTAINER_ENGINE=podman` to override. +- For rootless Podman: ensure the Podman socket is active (`systemctl --user start podman.socket`) and subuid/subgid ranges are configured (`sudo usermod --add-subuids 100000-165535 --add-subgids 100000-165535 $(whoami)`) - The `openshell` CLI must be available - For remote clusters: SSH access to the remote host @@ -300,10 +302,10 @@ Common issues: DNS misconfiguration is a common root cause, especially on remote/Linux hosts: ```bash -# Check the resolv.conf k3s is using +# Check the resolv.conf k3s is using for pod DNS openshell doctor exec -- cat /etc/rancher/k3s/resolv.conf -# Test DNS resolution from inside the container +# Test DNS from inside the container openshell doctor exec -- sh -c 'nslookup google.com || wget -q -O /dev/null http://google.com && echo "network ok" || echo "network unreachable"' ``` @@ -313,15 +315,23 @@ Check the entrypoint's DNS decision in the container logs: openshell doctor logs --lines 20 ``` -The entrypoint script selects DNS resolvers in this priority: +The entrypoint (`cluster-entrypoint.sh`) sets k3s pod DNS via a single strategy with one fallback: -1. Viable nameservers from `/etc/resolv.conf` (not loopback/link-local) -2. Docker `ExtServers` from `/etc/resolv.conf` comments -3. Host gateway IP (Docker Desktop only, `192.168.*`) -4. Fallback to `8.8.8.8` / `8.8.4.4` +1. **Docker DNS proxy** (`setup_dns_proxy()`): reads the `DOCKER_OUTPUT` iptables chain to discover where Docker's embedded DNS (`127.0.0.11`) is actually listening, installs DNAT rules so k3s pods can reach it via the container's `eth0` IP, and writes `nameserver ` to `/etc/rancher/k3s/resolv.conf`. On success logs: `Setting up DNS proxy: :53 -> 127.0.0.11`. +2. **Public DNS fallback**: if `setup_dns_proxy()` fails for any reason, logs `Warning: Could not discover Docker DNS ports from iptables` and writes `nameserver 8.8.8.8` / `nameserver 8.8.4.4` to `/etc/rancher/k3s/resolv.conf`. + +After either path, `verify_dns()` runs `nslookup` (5 retries) against the configured registry host (default `ghcr.io`). On failure it emits `DNS_PROBE_FAILED` into the logs. The Rust-side bootstrap (`runtime.rs` / `lib.rs`) watches for this marker and aborts early rather than spinning for the full 6-minute timeout. + +**Important:** there are two independent DNS paths inside the cluster container. The entrypoint only writes `/etc/rancher/k3s/resolv.conf` (pod DNS). The container's system `/etc/resolv.conf` (used by containerd for image pulls and by `nslookup`) is set by Docker or Podman at container start and is never touched by the entrypoint. These can point at different nameservers. + +**Under Podman:** `setup_dns_proxy()` always fails — Podman does not create a `DOCKER_OUTPUT` chain. k3s pod DNS always falls back to `8.8.8.8`/`8.8.4.4`. The cluster container runs on a named Podman bridge network, which uses **netavark + aardvark-dns**. Aardvark-dns listens on the bridge gateway IP (e.g. `10.89.x.1`) and forwards external queries to the host resolver. Podman sets the container's system `/etc/resolv.conf` to that address — so `nslookup ghcr.io` works fine even when `8.8.8.8` is blocked. This means **`DNS_PROBE_FAILED` is never emitted under Podman** even when pod-level DNS is broken: the entrypoint's `verify_dns()` and the Rust-side `probe_container_dns()` both call bare `nslookup`, which hits aardvark-dns via the system resolv.conf, not the k3s resolv.conf. Pod DNS failures surface later as CoreDNS upstream forwarding timeouts, not as an early bootstrap abort. + +To debug Podman pod DNS failures: check `/etc/rancher/k3s/resolv.conf` confirms `8.8.8.8` is there, then verify `8.8.8.8:53` UDP is reachable from the host with `nc -vzu 8.8.8.8 53`. If DNS is broken, all image pulls from the distribution registry will fail, as will pods that need external network access. +**Sandbox agent DNS is a separate enforcement layer.** The cluster DNS above controls what k3s pods and containerd can resolve. It has no bearing on what agent workloads inside sandboxes can reach. The sandbox supervisor (`openshell-sandbox`) creates an isolated Linux network namespace for each agent process with a veth pair, then installs iptables rules inside that namespace that REJECT all outbound UDP — including port 53 — except traffic destined for the supervisor's CONNECT proxy. Agent workloads cannot make raw DNS queries regardless of what nameservers are configured anywhere in the cluster. DNS must go through the HTTP CONNECT proxy. This is a kernel-enforced boundary at the netns level, not a configuration setting. The bypass monitor detects and logs any direct DNS attempt with the hint: `"DNS queries should route through the sandbox proxy; check resolver configuration"`. + ## Common Failure Patterns | Symptom | Likely Cause | Fix | @@ -329,7 +339,7 @@ If DNS is broken, all image pulls from the distribution registry will fail, as w | `tls handshake eof` from `openshell status` | Server not running or mTLS credentials missing/mismatched | Check StatefulSet replicas (Step 3) and mTLS files (Step 6) | | StatefulSet `0/0` replicas | StatefulSet scaled to zero (failed deploy, manual scale-down, or Helm misconfiguration) | `openshell doctor exec -- kubectl -n openshell scale statefulset openshell --replicas=1` | | Local mTLS files missing | Deploy was interrupted before credentials were persisted | Extract from cluster secret `openshell-client-tls` (Step 6) | -| Container not found | Image not built | `mise run docker:build:cluster` (local) or re-deploy (remote) | +| Container not found | Image not built | `mise run docker:build:cluster` (local, works with both Docker and Podman) or re-deploy (remote) | | Container exited, OOMKilled | Insufficient memory | Increase host memory or reduce workload | | Container exited, non-zero exit | k3s crash, port conflict, privilege issue | Check `openshell doctor logs` for details | | `/readyz` fails | k3s still starting or crashed | Wait longer or check container logs for k3s errors | @@ -343,10 +353,15 @@ If DNS is broken, all image pulls from the distribution registry will fail, as w | mTLS mismatch after redeploy | PKI rotated but workload not restarted, or rollout failed | Check that all three TLS secrets exist and that the openshell pod restarted after cert rotation (Step 6) | | Helm install job failed | Chart values error or dependency issue | `openshell doctor exec -- kubectl -n kube-system logs -l job-name=helm-install-openshell` | | NFD/GFD DaemonSets present (`node-feature-discovery`, `gpu-feature-discovery`) | Cluster was deployed before NFD/GFD were disabled (pre-simplify-device-plugin change) | These are harmless but add overhead. Clean up: `openshell doctor exec -- kubectl delete daemonset -n nvidia-device-plugin -l app.kubernetes.io/name=node-feature-discovery` and similarly for GFD. The `nvidia.com/gpu.present` node label is no longer applied; device plugin scheduling no longer requires it. | +| Podman socket not found | Rootless Podman service not started | `systemctl --user start podman.socket` and verify with `podman info` | +| Container creation fails with subuid/subgid error (Podman) | Missing user namespace ID mappings | `sudo usermod --add-subuids 100000-165535 --add-subgids 100000-165535 $(whoami)` then `podman system migrate` | +| Cgroups v1 detected (Podman) | Podman driver requires unified cgroup hierarchy | Set `systemd.unified_cgroup_hierarchy=1` kernel parameter and reboot | +| `--restart=always` ignored (Podman rootless) | Rootless Podman does not support `--restart=always` for containers | Use a systemd user service instead: `loginctl enable-linger $(whoami)` then create a `~/.config/systemd/user/` unit | | Architecture mismatch (remote) | Built on arm64, deploying to amd64 | Cross-build the image for the target architecture | | Port conflict | Another service on the configured gateway host port (default 8080) | Stop conflicting service or use `--port` on `openshell gateway start` to pick a different host port | | gRPC connect refused to `127.0.0.1:443` in CI | Docker daemon is remote (`DOCKER_HOST=tcp://...`) but metadata still points to loopback | Verify metadata endpoint host matches `DOCKER_HOST` and includes non-loopback host | -| DNS failures inside container | Entrypoint DNS detection failed | `openshell doctor exec -- cat /etc/rancher/k3s/resolv.conf` and `openshell doctor logs --lines 20` | +| DNS failures inside container (Docker) | `setup_dns_proxy()` failed to find `DOCKER_OUTPUT` iptables chain | `openshell doctor logs --lines 20` for `Warning: Could not discover Docker DNS ports`; try `docker network prune -f` and redeploy | +| Pod external name resolution fails (Podman) | `setup_dns_proxy()` always fails under Podman; k3s resolv.conf falls back to `8.8.8.8`/`8.8.4.4`, which is blocked on this network | `DNS_PROBE_FAILED` will NOT appear — entrypoint and Rust-side probes resolve via aardvark-dns (system `/etc/resolv.conf`), not the k3s resolv.conf; check `openshell doctor exec -- cat /etc/rancher/k3s/resolv.conf` to confirm fallback; verify `8.8.8.8:53` UDP reachable from host via `nc -vzu 8.8.8.8 53` | | Pods can't reach kube-dns / ClusterIP services | `br_netfilter` not loaded; bridge traffic bypasses iptables DNAT rules | `sudo modprobe br_netfilter` on the host, then `echo br_netfilter \| sudo tee /etc/modules-load.d/br_netfilter.conf` to persist. Known to be required on Jetson Linux 5.15-tegra; other kernels (e.g. standard x86/aarch64 Linux) may have bridge netfilter built in and work without the module. The entrypoint logs a warning when `/proc/sys/net/bridge/bridge-nf-call-iptables` is absent but does not abort — only act on it if DNS or service connectivity is actually broken. | | Node DiskPressure / MemoryPressure / PIDPressure | Insufficient disk, memory, or PIDs on host | Free disk (`docker system prune -a --volumes`), increase memory, or expand host resources. Bootstrap auto-detects via `HEALTHCHECK_NODE_PRESSURE` marker | | Pods evicted with "The node had condition: [DiskPressure]" | Host disk full, kubelet evicting pods | Free disk space on host, then `openshell gateway destroy && openshell gateway start` | diff --git a/Cargo.lock b/Cargo.lock index 1253ffffc8..3f50e57b4b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -232,9 +232,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.16.2" +version = "1.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc" +checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f" dependencies = [ "aws-lc-sys", "untrusted 0.7.1", @@ -243,9 +243,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.39.1" +version = "0.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a25cf98105baa966497416dbd42565ce3a8cf8dbfd59803ec9ad46f3126399" +checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7" dependencies = [ "cc", "cmake", @@ -282,9 +282,9 @@ dependencies = [ [[package]] name = "axum" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ "axum-core 0.5.6", "base64 0.22.1", @@ -309,7 +309,7 @@ dependencies = [ "sha1 0.10.6", "sync_wrapper", "tokio", - "tokio-tungstenite 0.28.0", + "tokio-tungstenite 0.29.0", "tower 0.5.3", "tower-layer", "tower-service", @@ -363,7 +363,7 @@ checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" dependencies = [ "getrandom 0.2.17", "instant", - "rand 0.8.5", + "rand 0.8.6", ] [[package]] @@ -451,9 +451,9 @@ dependencies = [ [[package]] name = "bitflags" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" dependencies = [ "serde_core", ] @@ -611,9 +611,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.59" +version = "1.2.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7a4d3ec6524d28a329fc53654bbadc9bdd7b0431f5d65f1a56ffb28a1ee5283" +checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" dependencies = [ "find-msvc-tools", "jobserver", @@ -690,9 +690,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.0" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" dependencies = [ "clap_builder", "clap_derive", @@ -712,9 +712,9 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.6.0" +version = "4.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19c9f1dde76b736e3681f28cec9d5a61299cbaae0fce80a68e43724ad56031eb" +checksum = "3ff7a1dccbdd8b078c2bdebff47e404615151534d5043da397ec50286816f9cb" dependencies = [ "clap", "clap_lex", @@ -724,9 +724,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.0" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" dependencies = [ "heck", "proc-macro2", @@ -838,7 +838,7 @@ checksum = "0940496e5c83c54f3b753d5317daec82e8edac71c33aaa1f666d76f518de2444" dependencies = [ "hax-lib", "pastey", - "rand 0.9.2", + "rand 0.9.4", ] [[package]] @@ -1944,9 +1944,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.7" +version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ "http", "hyper", @@ -1954,11 +1954,10 @@ dependencies = [ "log", "rustls", "rustls-native-certs", - "rustls-pki-types", "tokio", "tokio-rustls", "tower-service", - "webpki-roots 1.0.6", + "webpki-roots 1.0.7", ] [[package]] @@ -2337,9 +2336,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.94" +version = "0.3.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e04e2ef80ce82e13552136fabeef8a5ed1f985a96805761cbb9a2c34e7664d9" +checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" dependencies = [ "cfg-if", "futures-util", @@ -2520,15 +2519,15 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "libbz2-rs-sys" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" +checksum = "b3a6a8c165077efc8f3a971534c50ea6a1a18b329ef4a66e897a7e3a1494565f" [[package]] name = "libc" -version = "0.2.184" +version = "0.2.185" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" +checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" [[package]] name = "libcrux-intrinsics" @@ -2552,7 +2551,7 @@ dependencies = [ "libcrux-secrets", "libcrux-sha3", "libcrux-traits", - "rand 0.9.2", + "rand 0.9.4", "tls_codec", ] @@ -2593,7 +2592,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9adfd58e79d860f6b9e40e35127bfae9e5bd3ade33201d1347459011a2add034" dependencies = [ "libcrux-secrets", - "rand 0.9.2", + "rand 0.9.4", ] [[package]] @@ -2935,7 +2934,7 @@ checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ "num-integer", "num-traits", - "rand 0.8.5", + "rand 0.8.6", ] [[package]] @@ -2949,7 +2948,7 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand 0.8.5", + "rand 0.8.6", "serde", "smallvec", "zeroize", @@ -3160,6 +3159,29 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "openshell-driver-podman" +version = "0.0.0" +dependencies = [ + "clap", + "futures", + "http-body-util", + "hyper", + "hyper-util", + "miette", + "nix", + "openshell-core", + "serde", + "serde_json", + "temp-env", + "thiserror 2.0.18", + "tokio", + "tokio-stream", + "tonic", + "tracing", + "tracing-subscriber", +] + [[package]] name = "openshell-driver-vm" version = "0.0.0" @@ -3286,7 +3308,7 @@ dependencies = [ "tracing-appender", "tracing-subscriber", "uuid", - "webpki-roots 1.0.6", + "webpki-roots 1.0.7", ] [[package]] @@ -3294,7 +3316,7 @@ name = "openshell-server" version = "0.0.0" dependencies = [ "anyhow", - "axum 0.8.8", + "axum 0.8.9", "bytes", "clap", "futures", @@ -3313,6 +3335,7 @@ dependencies = [ "miette", "openshell-core", "openshell-driver-kubernetes", + "openshell-driver-podman", "openshell-ocsf", "openshell-policy", "openshell-router", @@ -3320,7 +3343,7 @@ dependencies = [ "pin-project-lite", "prost", "prost-types", - "rand 0.9.2", + "rand 0.9.4", "rcgen", "reqwest", "russh", @@ -3478,7 +3501,7 @@ dependencies = [ "delegate", "futures", "log", - "rand 0.8.5", + "rand 0.8.6", "sha2 0.10.9", "thiserror 1.0.69", "tokio", @@ -3646,7 +3669,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "rand 0.8.5", + "rand 0.8.6", ] [[package]] @@ -3735,9 +3758,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] name = "plain" @@ -3973,7 +3996,7 @@ dependencies = [ "bytes", "getrandom 0.3.4", "lru-slab", - "rand 0.9.2", + "rand 0.9.4", "ring", "rustc-hash", "rustls", @@ -4022,9 +4045,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -4033,9 +4056,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -4195,7 +4218,7 @@ dependencies = [ "msvc_spectre_libs", "num-bigint", "num-traits", - "rand 0.9.2", + "rand 0.9.4", "serde", "serde_json", "serde_yaml", @@ -4240,7 +4263,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots 1.0.6", + "webpki-roots 1.0.7", ] [[package]] @@ -4350,7 +4373,7 @@ dependencies = [ "pkcs1 0.8.0-rc.4", "pkcs5", "pkcs8 0.10.2", - "rand 0.9.2", + "rand 0.9.4", "rand_core 0.10.0-rc-3", "rsa 0.10.0-rc.12", "russh-cryptovec", @@ -4442,9 +4465,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.37" +version = "0.23.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" +checksum = "69f9466fb2c14ea04357e91413efb882e2a6d4a406e625449bc0a5d360d53a21" dependencies = [ "log", "once_cell", @@ -4488,9 +4511,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.10" +version = "0.103.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" +checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06" dependencies = [ "ring", "rustls-pki-types", @@ -5081,7 +5104,7 @@ dependencies = [ "memchr", "once_cell", "percent-encoding", - "rand 0.8.5", + "rand 0.8.6", "rsa 0.9.10", "serde", "sha1 0.10.6", @@ -5119,7 +5142,7 @@ dependencies = [ "md-5", "memchr", "once_cell", - "rand 0.8.5", + "rand 0.8.6", "serde", "serde_json", "sha2 0.10.9", @@ -5272,6 +5295,12 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7401a30af6cb5818bb64852270bb722533397edcfc7344954a38f420819ece2" +[[package]] +name = "symlink" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" + [[package]] name = "syn" version = "1.0.109" @@ -5522,9 +5551,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.51.1" +version = "1.52.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f66bf9585cda4b724d3e78ab34b73fb2bbaba9011b9bfdf69dc836382ea13b8c" +checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6" dependencies = [ "bytes", "libc", @@ -5587,14 +5616,14 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d25a406cddcc431a75d3d9afc6a7c0f7428d4891dd973e4d54c56b46127bf857" +checksum = "8f72a05e828585856dacd553fba484c242c46e391fb0e58917c942ee9202915c" dependencies = [ "futures-util", "log", "tokio", - "tungstenite 0.28.0", + "tungstenite 0.29.0", ] [[package]] @@ -5669,7 +5698,7 @@ dependencies = [ "indexmap 1.9.3", "pin-project", "pin-project-lite", - "rand 0.8.5", + "rand 0.8.6", "slab", "tokio", "tokio-util", @@ -5758,11 +5787,12 @@ dependencies = [ [[package]] name = "tracing-appender" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "786d480bce6247ab75f005b14ae1624ad978d3029d9113f0a22fa1ac773faeaf" +checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" dependencies = [ "crossbeam-channel", + "symlink", "thiserror 2.0.18", "time", "tracing-subscriber", @@ -5848,7 +5878,7 @@ dependencies = [ "http", "httparse", "log", - "rand 0.9.2", + "rand 0.9.4", "rustls", "rustls-pki-types", "sha1 0.10.6", @@ -5858,19 +5888,18 @@ dependencies = [ [[package]] name = "tungstenite" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442" +checksum = "6c01152af293afb9c7c2a57e4b559c5620b421f6d133261c60dd2d0cdb38e6b8" dependencies = [ "bytes", "data-encoding", "http", "httparse", "log", - "rand 0.9.2", + "rand 0.9.4", "sha1 0.10.6", "thiserror 2.0.18", - "utf-8", ] [[package]] @@ -6019,9 +6048,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.23.0" +version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9" +checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" dependencies = [ "getrandom 0.4.2", "js-sys", @@ -6063,11 +6092,11 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasip2" -version = "1.0.2+wasi-0.2.9" +version = "1.0.3+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.57.1", ] [[package]] @@ -6076,7 +6105,7 @@ version = "0.4.0+wasi-0.3.0-rc-2026-01-06" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" dependencies = [ - "wit-bindgen", + "wit-bindgen 0.51.0", ] [[package]] @@ -6087,9 +6116,9 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.117" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0551fc1bb415591e3372d0bc4780db7e587d84e2a7e79da121051c5c4b89d0b0" +checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89" dependencies = [ "cfg-if", "once_cell", @@ -6100,9 +6129,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.67" +version = "0.4.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03623de6905b7206edd0a75f69f747f134b7f0a2323392d664448bf2d3c5d87e" +checksum = "f371d383f2fb139252e0bfac3b81b265689bf45b6874af544ffa4c975ac1ebf8" dependencies = [ "js-sys", "wasm-bindgen", @@ -6110,9 +6139,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.117" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fbdf9a35adf44786aecd5ff89b4563a90325f9da0923236f6104e603c7e86be" +checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -6120,9 +6149,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.117" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dca9693ef2bab6d4e6707234500350d8dad079eb508dca05530c85dc3a529ff2" +checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904" dependencies = [ "bumpalo", "proc-macro2", @@ -6133,9 +6162,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.117" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39129a682a6d2d841b6c429d0c51e5cb0ed1a03829d8b3d1e69a011e62cb3d3b" +checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129" dependencies = [ "unicode-ident", ] @@ -6176,9 +6205,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.94" +version = "0.3.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd70027e39b12f0849461e08ffc50b9cd7688d942c1c8e3c7b22273236b4dd0a" +checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d" dependencies = [ "js-sys", "wasm-bindgen", @@ -6200,14 +6229,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.6", + "webpki-roots 1.0.7", ] [[package]] name = "webpki-roots" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cfaf3c063993ff62e73cb4311efde4db1efb31ab78a3e5c457939ad5cc0bed" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" dependencies = [ "rustls-pki-types", ] @@ -6617,6 +6646,12 @@ dependencies = [ "wit-bindgen-rust-macro", ] +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + [[package]] name = "wit-bindgen-core" version = "0.51.0" diff --git a/architecture/podman-driver.md b/architecture/podman-driver.md new file mode 100644 index 0000000000..3f155ecfac --- /dev/null +++ b/architecture/podman-driver.md @@ -0,0 +1,259 @@ +# Podman Compute Driver + +The Podman compute driver manages sandbox containers via the Podman REST API over a Unix socket. It targets single-machine and developer environments where rootless container isolation is preferred over a full Kubernetes cluster. The driver runs in-process within the gateway server and delegates all sandbox isolation enforcement to the `openshell-sandbox` supervisor binary, which is sideloaded into each container via an OCI image volume mount. + +## Source File Index + +All paths are relative to `crates/openshell-driver-podman/src/`. + +| File | Purpose | +|------|---------| +| `lib.rs` | Crate root; declares modules and re-exports `PodmanComputeConfig`, `PodmanComputeDriver`, `ComputeDriverService` | +| `main.rs` | Standalone binary entrypoint; parses CLI args/env vars, constructs the driver, starts a gRPC server with graceful shutdown | +| `driver.rs` | Core `PodmanComputeDriver` -- sandbox lifecycle (create/stop/delete/list/get), endpoint resolution, GPU detection, rootless pre-flight checks | +| `client.rs` | `PodmanClient` -- async HTTP/1.1 client over Unix socket for the Podman libpod REST API (containers, volumes, networks, secrets, images, events, system info) | +| `container.rs` | Container spec construction -- labels, env vars, resource limits, capabilities, seccomp config, health checks, port mappings, image volumes, secret injection | +| `config.rs` | `PodmanComputeConfig` struct, `ImagePullPolicy` enum, default socket path resolution, `Debug` impl that redacts secrets | +| `grpc.rs` | `ComputeDriverService` -- tonic gRPC service mapping RPCs to driver methods, with error-to-Status conversion | +| `watcher.rs` | Watch stream -- initial state sync via container list, then live Podman events mapped to `WatchSandboxesEvent` protobuf messages | + +## Architecture + +The Podman driver is one of three `ComputeDriver` implementations. It communicates with the Podman daemon over a Unix socket and delegates sandbox isolation to the supervisor binary running inside each container. + +```mermaid +graph TB + CLI["openshell CLI"] -->|gRPC| GW["Gateway Server
(openshell-server)"] + GW -->|in-process| PD["PodmanComputeDriver"] + PD -->|HTTP/1.1
Unix socket| PA["Podman API"] + PA -->|OCI runtime
crun/runc| C["Sandbox Container"] + C -->|image volume
read-only| SV["Supervisor Binary
/opt/openshell/bin/openshell-sandbox"] + SV -->|creates| NS["Nested Network Namespace
veth pair + proxy"] + SV -->|enforces| LL["Landlock + seccomp"] + SV -->|gRPC callback| GW +``` + +### Driver Comparison + +| Aspect | Kubernetes | VM (libkrun) | Podman | +|--------|-----------|--------------|--------| +| Execution model | In-process | Standalone subprocess (gRPC over UDS) | In-process | +| Backend | K8s API (CRD + controller) | libkrun hypervisor (KVM/HVF) | Podman REST API (Unix socket) | +| Isolation boundary | Container (supervisor inside pod) | Hardware VM | Container (supervisor inside container) | +| Supervisor delivery | hostPath volume (read-only) | Embedded in rootfs tarball | OCI image volume (read-only) | +| Network model | Supervisor creates netns inside pod | gvproxy virtio-net (192.168.127.0/24) | Supervisor creates netns inside container | +| Credential injection | Plaintext env var + K8s Secret volume (0400) | Rootfs file copy (0600) + env vars | Podman `secret_env` API + env vars | +| GPU support | Yes (nvidia.com/gpu resource) | No | Yes (CDI device) | +| `stop_sandbox` | Unimplemented | Unimplemented | Implemented (graceful stop) | +| State storage | Kubernetes API (CRD) | In-memory HashMap + filesystem | Podman daemon (container state) | +| Endpoint resolution | Pod IP / cluster DNS | 127.0.0.1 + allocated port | 127.0.0.1 + ephemeral port | + +## Isolation Model + +The Podman driver provides the same four protection layers as the Kubernetes driver. The driver itself does not implement isolation primitives directly -- it configures the container so that the `openshell-sandbox` supervisor binary can enforce them at runtime. + +### Container Security Configuration + +The container spec (`container.rs`) sets: + +| Setting | Value | Rationale | +|---------|-------|-----------| +| `user` | `0:0` | Supervisor needs root for namespace creation, proxy setup, Landlock/seccomp | +| `cap_drop` | `ALL` | Drop all capabilities, then selectively add back | +| `cap_add` | `SYS_ADMIN, NET_ADMIN, SYS_PTRACE, SYSLOG, SETUID, SETGID, DAC_READ_SEARCH` | See capability breakdown below | +| `no_new_privileges` | `true` | Prevent privilege escalation after exec | +| `seccomp_profile_path` | `unconfined` | Supervisor installs its own policy-aware BPF filter; container-level profile would block Landlock/seccomp syscalls during setup | + +### Capability Breakdown + +The Kubernetes driver uses 4 capabilities (`SYS_ADMIN, NET_ADMIN, SYS_PTRACE, SYSLOG`). The Podman driver adds 3 more, all required for rootless operation: + +| Capability | Shared with K8s? | Purpose | +|------------|------------------|---------| +| `SYS_ADMIN` | Yes | seccomp filter installation, namespace creation, Landlock | +| `NET_ADMIN` | Yes | Network namespace veth setup, IP/route configuration | +| `SYS_PTRACE` | Yes | Reading `/proc//exe` and ancestor walk for binary identity | +| `SYSLOG` | Yes | Reading `/dev/kmsg` for bypass-detection diagnostics | +| `SETUID` | Podman only | `drop_privileges()` calls `setuid()` to sandbox user. Rootless `cap_drop: ALL` removes this from the bounding set. | +| `SETGID` | Podman only | `drop_privileges()` calls `setgid()` + `initgroups()`. Same rootless reason as SETUID. | +| `DAC_READ_SEARCH` | Podman only | Proxy reads `/proc//fd/` across UIDs for binary identity. In rootless Podman, supervisor (UID 0 in user namespace) and sandbox processes have different UIDs. | + +In the Kubernetes driver, these three capabilities are implicitly available because the kubelet does not drop them from the bounding set. In rootless Podman, `cap_drop: ALL` removes everything, requiring explicit re-addition. + +All capabilities are only available to the supervisor process. Sandbox child processes lose them after `setuid()` to the sandbox user in the `pre_exec` hook. + +## Supervisor Sideloading + +The supervisor binary is delivered to sandbox containers via Podman's OCI image volume mechanism, distinct from both the Kubernetes hostPath approach and the VM's embedded rootfs. + +```mermaid +sequenceDiagram + participant D as PodmanComputeDriver + participant P as Podman API + participant C as Sandbox Container + + D->>P: pull_image("openshell/supervisor:latest", "missing") + D->>P: create_container(spec with image_volumes) + Note over P: Podman resolves image_volumes at
libpod layer before OCI spec generation + P->>C: Mount supervisor image at /opt/openshell/bin (read-only) + D->>P: start_container + C->>C: entrypoint: /opt/openshell/bin/openshell-sandbox +``` + +The supervisor image is a `FROM scratch` image containing only the `openshell-sandbox` binary. It is built by the `supervisor-output` target in `deploy/docker/Dockerfile.images`. The `image_volumes` field in the container spec mounts this image's filesystem at `/opt/openshell/bin` with `rw: false`, making it a read-only overlay that the sandbox cannot tamper with. + +## Network Model + +Sandbox network isolation uses a two-layer approach: a Podman bridge network for container-to-host communication, and a nested network namespace (created by the supervisor) for sandbox process isolation. + +```mermaid +graph TB + subgraph Host + GW["Gateway Server
127.0.0.1:8080"] + PS["Podman Socket"] + end + + subgraph Bridge["Podman Bridge Network (10.89.x.x)"] + subgraph Container["Sandbox Container"] + SV["Supervisor
(root in user ns)"] + subgraph NestedNS["Nested Network Namespace"] + SP["Sandbox Process
(sandbox user)"] + VE2["veth1: 10.200.0.2"] + end + VE1["veth0: 10.200.0.1
(CONNECT proxy)"] + SV --- VE1 + VE1 ---|veth pair| VE2 + end + end + + GW -.->|SSH via supervisor relay
gRPC session| SV + SV -->|gRPC callback via
host.containers.internal| GW + SP -->|all egress via proxy| VE1 +``` + +Key points: + +- **Bridge network**: Created by `client.ensure_network()` with DNS enabled. Containers on the bridge can see each other at L3, but sandbox processes cannot because they are isolated inside the nested netns. +- **Nested netns**: The supervisor creates a private `NetworkNamespace` with a veth pair (10.200.0.1/24 <-> 10.200.0.2/24). Sandbox processes enter this netns via `setns(fd, CLONE_NEWNET)` in the `pre_exec` hook, forcing all traffic through the CONNECT proxy. +- **Port publishing**: SSH uses `host_port: 0` (ephemeral port assignment) for health checks and debug access. The gateway SSH tunnel uses the supervisor relay (`supervisor_sessions.open_relay()`) rather than connecting directly to the published port. +- **Host gateway**: `host.containers.internal:host-gateway` in `/etc/hosts` allows containers to reach the gateway server on the host. +- **nsenter**: The supervisor uses `nsenter --net=` instead of `ip netns exec` for namespace operations, avoiding the sysfs remount that fails in rootless containers. + +## Supervisor relay (SSH Unix socket) + +Podman now follows the same end-to-end contract as the Kubernetes and VM drivers for the in-container SSH relay: **gateway config → `PodmanComputeConfig` → sandbox environment → supervisor session registration on that path**. + +1. `openshell-core` `Config::sandbox_ssh_socket_path` (gateway YAML / defaults) is copied into `PodmanComputeConfig::sandbox_ssh_socket_path` when the gateway builds the in-process driver (`crates/openshell-server/src/lib.rs`, `ComputeDriverKind::Podman`). +2. `build_env()` in `container.rs` sets `OPENSHELL_SSH_SOCKET_PATH` to that value, alongside required vars `OPENSHELL_ENDPOINT` and `OPENSHELL_SANDBOX_ID` (and `OPENSHELL_SANDBOX`, etc.). These driver-controlled entries overwrite user template env to prevent spoofing. +3. The supervisor reads `OPENSHELL_SSH_SOCKET_PATH` and uses it for the Unix socket the gateway’s SSH stack bridges to. The standalone `openshell-driver-podman` binary sets the same struct field from `OPENSHELL_SANDBOX_SSH_SOCKET_PATH` (`main.rs`). + +## Credential Injection + +The SSH handshake secret is injected via Podman's `secret_env` API rather than a plaintext environment variable. + +| Credential | Mechanism | Visible in `inspect`? | Visible in `/proc//environ`? | +|------------|-----------|----------------------|----------------------------------| +| SSH handshake secret | Podman `secret_env` (created via secrets API, referenced by name) | No | Yes (supervisor only; scrubbed from children) | +| Sandbox identity (`OPENSHELL_SANDBOX_ID`, etc.) | Plaintext env var | Yes | Yes | +| gRPC endpoint (`OPENSHELL_ENDPOINT`) | Plaintext env var, override-protected | Yes | Yes | +| Supervisor relay socket path (`OPENSHELL_SSH_SOCKET_PATH`) | Plaintext env var, override-protected (same value as `PodmanComputeConfig::sandbox_ssh_socket_path`) | Yes | Yes | + +The `build_env()` function in `container.rs` inserts user-supplied variables first, then unconditionally overwrites all security-critical variables to prevent spoofing via sandbox templates: `OPENSHELL_SANDBOX`, `OPENSHELL_SANDBOX_ID`, `OPENSHELL_ENDPOINT`, `OPENSHELL_SSH_SOCKET_PATH`, `OPENSHELL_SSH_LISTEN_ADDR`, `OPENSHELL_SSH_HANDSHAKE_SKEW_SECS`, `OPENSHELL_CONTAINER_IMAGE`, `OPENSHELL_SANDBOX_COMMAND`. + +The `PodmanComputeConfig::Debug` impl redacts the handshake secret as `[REDACTED]`. + +## Sandbox Lifecycle + +### Creation Flow + +```mermaid +sequenceDiagram + participant GW as Gateway + participant D as PodmanComputeDriver + participant P as Podman API + + GW->>D: create_sandbox(DriverSandbox) + D->>D: validate name + id + D->>D: validated_container_name() + + D->>P: pull_image(supervisor, "missing") + D->>P: pull_image(sandbox_image, policy) + + D->>P: create_secret(handshake) + Note over D: On failure below, rollback secret + + D->>P: create_volume(workspace) + Note over D: On failure below, rollback volume + secret + + D->>P: create_container(spec) + alt Conflict (409) + D->>P: remove_volume + remove_secret + D-->>GW: AlreadyExists + end + Note over D: On failure below, rollback container + volume + secret + + D->>P: start_container + D-->>GW: Ok +``` + +Each step rolls back all previously-created resources on failure. The Conflict path (409 from container creation) cleans up the volume and secret because they are keyed by the new sandbox's ID, not the conflicting container's. + +### Readiness and health + +The container `healthconfig` in `container.rs` marks the sandbox healthy when **any** of these signals succeeds: legacy file marker `/var/run/openshell-ssh-ready`, **or** `test -S` on the configured supervisor Unix socket path (`sandbox_ssh_socket_path` / `OPENSHELL_SSH_SOCKET_PATH`), **or** the prior TCP check (`ss` listening on the in-container SSH port). That allows relay-only readiness when the supervisor exposes the socket without the old marker or published-port signal. + +### Deletion Flow + +1. Validate `sandbox_name` and stable `sandbox_id` from `DeleteSandboxRequest` +2. Best-effort inspect cross-checks the container label when present, but cleanup remains keyed by the request `sandbox_id` +3. Best-effort stop (result ignored) +4. Force-remove container (`?force=true&v=true`) +5. Remove workspace volume derived from the request `sandbox_id` (warn on failure, continue) +6. Remove handshake secret derived from the request `sandbox_id` (warn on failure, continue) + +If the container is already gone during inspect or remove, the driver still performs idempotent volume/secret cleanup using the request `sandbox_id` and returns `Ok(false)` for the container-delete result. This prevents leaked Podman resources after out-of-band container removal or label drift. + +## Configuration + +| Environment Variable | CLI Flag | Default | Description | +|---------------------|----------|---------|-------------| +| `OPENSHELL_PODMAN_SOCKET` | `--podman-socket` | `$XDG_RUNTIME_DIR/podman/podman.sock` | Podman API Unix socket path | +| `OPENSHELL_SANDBOX_IMAGE` | `--sandbox-image` | (from gateway config) | Default OCI image for sandboxes | +| `OPENSHELL_SANDBOX_IMAGE_PULL_POLICY` | `--sandbox-image-pull-policy` | `missing` | Pull policy: `always`, `missing`, `never`, `newer` | +| `OPENSHELL_GRPC_ENDPOINT` | `--grpc-endpoint` | Auto-detected via `host.containers.internal` | Gateway gRPC endpoint for sandbox callbacks | +| `OPENSHELL_NETWORK_NAME` | `--network-name` | `openshell` | Podman bridge network name | +| `OPENSHELL_SANDBOX_SSH_PORT` | `--sandbox-ssh-port` | `2222` | SSH port inside the container | +| `OPENSHELL_SSH_HANDSHAKE_SECRET` | `--ssh-handshake-secret` | (required) | Shared secret for NSSH1 handshake | +| `OPENSHELL_SANDBOX_SSH_SOCKET_PATH` | `--sandbox-ssh-socket-path` | `/run/openshell/ssh.sock` | Standalone driver only: supervisor Unix socket path in `PodmanComputeConfig` (in-gateway Podman uses server `config.sandbox_ssh_socket_path`) | +| `OPENSHELL_STOP_TIMEOUT` | `--stop-timeout` | `10` | Container stop timeout in seconds (SIGTERM -> SIGKILL) | +| `OPENSHELL_SUPERVISOR_IMAGE` | `--supervisor-image` | `openshell/supervisor:latest` (struct default; standalone binary requires explicit value) | OCI image containing the supervisor binary | + +## Rootless-Specific Adaptations + +The Podman driver is designed for rootless operation. The following adaptations were made compared to the Kubernetes driver: + +1. **subuid/subgid pre-flight check**: `check_subuid_range()` in `driver.rs` warns operators if `/etc/subuid` or `/etc/subgid` entries are missing for the current user. Not a hard error because some systems use LDAP or other mechanisms. + +2. **cgroups v2 requirement**: The driver refuses to start if cgroups v1 is detected. Rootless Podman requires the unified cgroup hierarchy. + +3. **nsenter for namespace operations**: `run_ip_netns()` and `run_iptables_netns()` in `crates/openshell-sandbox/src/sandbox/linux/netns.rs` use `nsenter --net=` instead of `ip netns exec` to avoid the sysfs remount that requires real `CAP_SYS_ADMIN` in the host user namespace. + +4. **DAC_READ_SEARCH capability**: Required for the proxy to read `/proc//fd/` across UIDs within the user namespace. + +5. **SETUID/SETGID capabilities**: Required for `drop_privileges()` to call `setuid()`/`setgid()` after `cap_drop: ALL` removes them from the bounding set. + +6. **host.containers.internal**: Used instead of Docker's `host.docker.internal` for container-to-host communication. Injected via `hostadd` with Podman's `host-gateway` magic value. + +7. **Ephemeral port publishing**: SSH port uses `host_port: 0` because the bridge network IP (10.89.x.x) is not routable from the host in rootless mode. The published port is used for health checks and debug access; the gateway SSH tunnel uses the supervisor relay. + +8. **tmpfs at `/run/netns`**: A private tmpfs is mounted so the supervisor can create named network namespaces via `ip netns add`, which requires `/run/netns` to exist and be writable. + +## Implementation References + +- Gateway integration: `crates/openshell-server/src/compute/mod.rs` (`new_podman` and `PodmanComputeDriver` wiring) +- Server configuration: `crates/openshell-server/src/lib.rs` (`ComputeDriverKind::Podman` — builds `PodmanComputeConfig` including `sandbox_ssh_socket_path` from gateway `Config`) +- Gateway relay path: `openshell-core` `Config::sandbox_ssh_socket_path` in `crates/openshell-core/src/config.rs` +- SSRF mitigation: `crates/openshell-core/src/net.rs` (IP classification: `is_always_blocked_ip`, `is_internal_ip`), `crates/openshell-sandbox/src/proxy.rs` (runtime enforcement on CONNECT/forward proxy), `crates/openshell-server/src/grpc/policy.rs` (load-time validation via `validate_rule_not_always_blocked`) +- Sandbox supervisor: `crates/openshell-sandbox/src/` (Landlock, seccomp, netns, proxy -- shared by all drivers) +- Container engine abstraction: `tasks/scripts/container-engine.sh` (build/deploy support for Docker and Podman) +- Supervisor image build: `deploy/docker/Dockerfile.images` (lines 183-184, `supervisor-output` target) diff --git a/architecture/podman-rootless-networking.md b/architecture/podman-rootless-networking.md new file mode 100644 index 0000000000..b267cfffac --- /dev/null +++ b/architecture/podman-rootless-networking.md @@ -0,0 +1,372 @@ +# Rootless Podman Networking + +Deep-dive into how networking works in the Podman compute driver when running rootless with pasta as the network backend. Covers the external tooling (Podman, Netavark, pasta, aardvark-dns), the three nested namespace layers, and the complete data paths for SSH, outbound traffic, and supervisor-to-gateway communication. + +For the general Podman driver architecture (lifecycle, API surface, driver comparison), see [podman-driver.md](podman-driver.md). + +## Component Stack + +Podman's networking is composed of four independent projects: + +| Component | Language | Role | +|-----------|----------|------| +| **Podman** | Go | Container runtime; orchestrates network lifecycle | +| **Netavark** | Rust | Network backend; creates interfaces, bridges, firewall rules | +| **aardvark-dns** | Rust | Authoritative DNS server for container name resolution (A/AAAA records) | +| **pasta** (part of passt) | C | User-mode networking; L2-to-L4 socket translation for rootless containers | + +The key split: rootful containers default to Netavark (bridge networking with real kernel interfaces), while rootless containers default to pasta (user-mode networking, no privileges needed). + +## How Netavark Works (Rootful) + +Netavark is invoked by Podman as an external binary. It reads a JSON network configuration from STDIN and executes one of three commands: + +- `netavark setup ` -- creates interfaces, assigns IPs, sets up firewall rules for NAT/port-forwarding +- `netavark teardown ` -- reverses setup; removes interfaces and firewall rules +- `netavark create` -- takes a partial network config and completes it (assigns subnets, gateways) + +For rootful bridge networking: + +1. Podman creates a network namespace for the container +2. Podman invokes `netavark setup` passing the network config JSON +3. Netavark creates a bridge (e.g., `podman0`) if it doesn't exist -- default subnet is `10.88.0.0/16` +4. Netavark creates a veth pair -- one end goes into the container's netns, the other attaches to the bridge +5. Netavark assigns an IP from the subnet to the container's veth interface (host-local IPAM) +6. Netavark configures iptables/nftables rules -- masquerade for outbound, DNAT for port mappings +7. Netavark starts aardvark-dns if DNS is enabled, listening on the bridge gateway address + +``` +Host Kernel + | + +-- Bridge interface (e.g., "podman0") <-- created by Netavark + | | + | +-- veth pair endpoint (host side, container 1) + | +-- veth pair endpoint (host side, container 2) + | + +-- Host physical interface (e.g., eth0) + | + +-- NAT (iptables/nftables rules managed by Netavark) +``` + +Netavark also supports macvlan networks (container gets a sub-interface of a physical host NIC with its own MAC, appearing directly on the physical network) and external plugins via a documented JSON API. + +## How Pasta Works (Rootless) + +### The Problem + +Unprivileged users cannot create network interfaces on the host. They cannot create veth pairs, bridges, or configure iptables rules. Netavark's bridge approach cannot work directly for rootless containers. + +### The Solution + +Pasta (part of the `passt` project -- same binary, different command name) operates entirely in userspace, translating between the container's L2 TAP interface and the host's L4 sockets. It requires no capabilities or privileges. + +``` +Container Network Namespace + | + +-- TAP device (e.g., "eth0") + | ^ + | | L2 frames (Ethernet) + | v + +-- pasta process (userspace) + | + | Translation: L2 frames <-> L4 sockets + | + v + Host Network Stack (native TCP/UDP/ICMP sockets) +``` + +### Detailed Data Path + +For an outbound TCP connection from a container: + +1. The application calls `connect()` to an external address +2. The kernel routes the packet through the default gateway to the TAP device +3. Pasta reads the raw Ethernet frame from the TAP file descriptor +4. Pasta parses L2/L3/L4 headers and identifies the TCP SYN +5. Pasta opens a native TCP socket on the host and calls `connect()` to the same destination +6. When the host socket connects, pasta reflects the SYN-ACK back through the TAP as an L2 frame +7. For ongoing data transfer, pasta translates between TAP frames and the host socket, coordinating TCP windows and acknowledgments between the two sides + +Pasta does NOT maintain per-connection packet buffers -- it reflects observed sending windows and ACKs directly between peers. This is a thinner translation layer than a full TCP/IP stack (like slirp4netns used). + +### Built-in Services + +Pasta includes minimalistic network services so the container's stack can auto-configure: + +| Service | Purpose | +|---------|---------| +| ARP proxy | Resolves the gateway address to the host's MAC address | +| DHCP server | Hands out a single IPv4 address (same as host's upstream interface) | +| NDP proxy | Handles IPv6 neighbor discovery, SLAAC prefix advertisement | +| DHCPv6 server | Hands out a single IPv6 address (same as host's upstream interface) | + +By default there is no NAT -- pasta copies the host's IP addresses into the container namespace. + +### Local Connection Bypass (Splice Path) + +For connections between the container and the host, pasta implements a zero-copy bypass: + +- Packets with a local destination skip L2 translation entirely +- `splice(2)` for TCP (zero-copy), `recvmmsg(2)` / `sendmmsg(2)` for UDP (batched) +- Achieves ~38 Gbps TCP throughput for local connections + +### Port Forwarding + +By default, pasta uses auto-detection: it scans `/proc/net/tcp` and `/proc/net/tcp6` periodically and automatically forwards any ports that are bound/listening. Port forwarding is fully configurable via pasta options. + +### Security Properties + +- No dynamic memory allocation (`sbrk`, `brk`, `mmap` blocked via seccomp) +- All capabilities dropped (except `CAP_NET_BIND_SERVICE` if granted) +- Restrictive seccomp profiles (43 syscalls allowed on x86_64) +- Detaches into its own user, mount, IPC, UTS, PID namespaces +- No external dependencies beyond libc +- ~5,000 lines of code target + +### Inter-Container Limitation + +Unlike bridge networking, pasta containers are isolated from each other by default. No virtual bridge connects them. Communication requires port mappings through the host, pods (shared network namespace), or opting into rootless Netavark bridge networking via `podman network create`. + +## Three Nested Namespaces in the Podman Driver + +The Podman compute driver creates three layers of network isolation: + +``` +Namespace 1: Host + | + pasta manages port forwarding (127.0.0.1:) + gateway listens on 0.0.0.0:8080 + | +Namespace 2: Rootless Podman network namespace (managed by pasta) + | + Bridge "openshell" (10.89.x.0/24) + aardvark-dns for container name resolution + | + Container netns (10.89.x.2) + supervisor, proxy, SSH daemon all run here + | +Namespace 3: Inner sandbox netns (created by supervisor) + | + veth pair (10.200.0.1 <-> 10.200.0.2) + iptables forces all traffic through proxy + user workload runs here +``` + +Pasta bridges namespace 1 and 2, the veth pair bridges namespace 2 and 3, and the proxy at the boundary of 2/3 enforces network policy. + +### Layer 1: Pasta (Rootless Podman Bridge) + +At driver startup (`driver.rs:104-114`), the driver ensures a Podman bridge network exists: + +```rust +client.ensure_network(&config.network_name).await?; +``` + +This creates a bridge network named `"openshell"` (default from `DEFAULT_NETWORK_NAME` in `openshell-core/src/config.rs`) with `dns_enabled: true`. In rootless mode, this bridge exists inside a user namespace managed by pasta. The bridge IP range (e.g., `10.89.x.x`) is not routable from the host. + +``` +Host (your machine) + | + 127.0.0.1: <--- pasta binds this on the host + | + [pasta process] <--- translates L4 sockets <-> L2 TAP frames + | + [rootless network namespace] + | + Bridge "openshell" (10.89.1.0/24) + | + +-- 10.89.1.1 (bridge gateway, aardvark-dns listens here) + | + +-- veth --> Container netns + | + 10.89.1.2 (container IP) +``` + +### Layer 2: Container Networking (Pasta Port Forwarding) + +The container spec (`container.rs:447-471`) configures: + +- `nsmode: "bridge"` -- uses the Podman bridge network +- `networks: {"openshell"}` -- attaches to the named bridge +- `portmappings: [{host_port: 0, container_port: 2222, protocol: "tcp"}]` -- publishes SSH on an ephemeral host port +- `hostadd: ["host.containers.internal:host-gateway"]` -- resolves to the host IP (pasta uses `169.254.1.2` in rootless mode) + +Pasta is never explicitly configured. The driver sets `nsmode: "bridge"` and Podman selects pasta automatically as the rootless network backend. The driver logs the detected backend at startup (`driver.rs:86`): + +```rust +network_backend = %info.host.network_backend, +``` + +The `host.containers.internal` hostname (the Podman equivalent of Docker's `host.docker.internal`) is injected into `/etc/hosts` so the supervisor can reach the gateway on the host. The gRPC callback endpoint is auto-detected at `driver.rs:116-130`: + +```rust +if config.grpc_endpoint.is_empty() { + config.grpc_endpoint = + format!("http://host.containers.internal:{}", config.gateway_port); +} +``` + +The bridge gateway IP does NOT work for this purpose in rootless mode because it lives inside the user namespace, not on the host. + +### Layer 3: Inner Sandbox Network Namespace + +Inside the container, the supervisor creates another network namespace (`netns.rs:53-178`, setup at lines 53-63, `ip netns add` at line 77) for the user workload: + +``` +Container (10.89.1.2 on the Podman bridge) + | + [Supervisor process - runs in container's default netns] + | + +-- Proxy listener at 10.200.0.1:3128 + | + +-- veth pair: veth-h-{short_id} <-> veth-s-{short_id} + | + +-- Inner network namespace "sandbox-{short_id}" (short_id = first 8 chars of UUID) + | + 10.200.0.2/24 + | + default route -> 10.200.0.1 (supervisor's proxy) + | + [User's code runs here] + | + iptables rules (IPv4; IPv6 installed best-effort): + ACCEPT -> 10.200.0.1:{proxy_port} TCP (proxy) + ACCEPT -> loopback (-o lo) + ACCEPT -> established/related (conntrack) + LOG -> TCP SYN bypass attempts (rate-limited 5/sec) + REJECT -> TCP (icmp-port-unreachable) + LOG -> UDP bypass attempts (rate-limited 5/sec) + REJECT -> UDP (icmp-port-unreachable) +``` + +The supervisor uses `nsenter --net=` rather than `ip netns exec` to avoid sysfs remount issues that arise under rootless Podman where real `CAP_SYS_ADMIN` is unavailable (`netns.rs:681-716`, function body at 691). + +A tmpfs is mounted at `/run/netns` in the container spec (`container.rs:458-463`) so the supervisor can create named network namespaces. In rootless Podman this directory does not exist on the host, so `mkdir` would fail with `EPERM` without a private tmpfs. + +## Complete Data Paths + +### SSH Session: Client to Sandbox Shell + +``` +Client (CLI on user's machine) + | + 1. gRPC: CreateSshSession -> gateway (returns token, connect_path) + 2. HTTP CONNECT /connect/ssh to gateway + (headers: x-sandbox-id, x-sandbox-token) + | +Gateway (host, port 8080) + | + 3. Looks up SupervisorSession for sandbox_id + 4. Sends RelayOpen{channel_id} over ConnectSupervisor bidi stream + | + [gRPC traverses: host -> pasta L4 translation -> container bridge] + | +Supervisor (inside container at 10.89.x.2) + | + 5. Receives RelayOpen, opens new RelayStream RPC back to gateway + 6. Sends RelayInit{channel_id} on the stream + 7. Connects to Unix socket /run/openshell/ssh.sock + 8. Bidirectional bridge: RelayStream <-> Unix socket (16 KiB chunks) + | +SSH daemon (inside container, Unix socket only, root-only permissions) + | + 9. Authenticates (all auth accepted -- access gated by relay chain) + 10. Spawns shell process + 11. Shell enters inner netns via setns(fd, CLONE_NEWNET) + | +User's shell (in sandbox netns at 10.200.0.2) +``` + +The SSH daemon listens on a Unix socket (not a TCP port) with 0600 permissions. The published port mapping (`host_port: 0 -> container_port: 2222`) exists in the container spec but is currently inert -- nothing listens on TCP 2222 inside the container. All SSH communication uses the gRPC reverse-connect relay pattern exclusively. + +### Outbound HTTP Request from Sandbox Process + +``` +User's code (inner netns, 10.200.0.2) + | + 1. curl https://api.example.com + (HTTP_PROXY=http://10.200.0.1:3128 set via environment) + | + 2. TCP connect to 10.200.0.1:3128 + (allowed by iptables -- only permitted egress destination) + | + 3. HTTP CONNECT api.example.com:443 + | +Supervisor proxy (10.200.0.1:3128 in container netns) + | + 4. OPA policy evaluation (process identity via /proc/net/tcp -> PID) + 5. SSRF check (block internal IPs unless allowed by policy) + 6. Optional L7: TLS intercept, HTTP method/path inspection + | + 7. If allowed: TCP connect to api.example.com:443 + (from container netns, 10.89.x.2) + | + 8. Through Podman bridge -> pasta L2-to-L4 -> host -> internet +``` + +### Supervisor gRPC Callback to Gateway + +``` +Supervisor (container netns, 10.89.x.2) + | + 1. gRPC connect to http://host.containers.internal:8080 + (resolves to 169.254.1.2:8080 via /etc/hosts) + | + 2. Routed through container default gateway (bridge) + | + 3. Pasta translates: L2 frame -> host L4 socket + (pasta host-gateway mapping: 169.254.1.2 -> 127.0.0.1) + | + 4. Host TCP socket connects to 127.0.0.1:8080 + | +Gateway (host, port 8080) + | + 5. ConnectSupervisor bidirectional stream established + 6. Heartbeats every N seconds (gateway sends interval in SessionAccepted, default 15s) + 7. Reconnects with exponential backoff (1s initial, 30s max) on failure + 8. Same gRPC channel reused for RelayStream calls (no new TLS handshake) +``` + +## Differences from the Kubernetes Driver + +| Aspect | Kubernetes | Podman (rootless pasta) | +|--------|-----------|----------------------| +| Container/Pod IP | Routable cluster-wide | Non-routable (10.89.x.x inside user namespace) | +| Network reachability | Pod IPs reachable from gateway | Bridge not routable from host; requires pasta port forwarding or `host.containers.internal` | +| Sandbox -> Gateway | Direct TCP to K8s service IP | `host.containers.internal` (169.254.1.2 via pasta) | +| SSH transport | Reverse gRPC relay (`ConnectSupervisor` + `RelayStream`) -- same mechanism as Podman | Reverse gRPC relay (`ConnectSupervisor` + `RelayStream`) | +| Port publishing | Not needed (routable IPs) | Ephemeral host port via pasta port forwarding | +| TLS | mTLS via K8s secrets | Disabled by default (loopback-only, `--disable-tls`) | +| DNS | Kubernetes CoreDNS | Podman bridge DNS (aardvark-dns, `dns_enabled: true`) | +| Network policy | K8s NetworkPolicy (ingress restricted to gateway) | iptables inside inner sandbox netns | +| Supervisor delivery | hostPath volume from k3s node | OCI image volume mount (FROM scratch image) | +| Secrets | K8s Secret volume mount (TLS certs); SSH handshake secret via env var | Podman `secret_env` API (hidden from `podman inspect`) | + +Both drivers use the same reverse gRPC relay (`ConnectSupervisor` + `RelayStream`) for SSH transport. The most significant difference is network reachability: in rootless Podman, the bridge network is not routable from the host, so all communication between host and container goes through either pasta port forwarding (`portmappings`) or the `host.containers.internal` hostname (resolved to `169.254.1.2` by pasta). + +## Port Assignments + +| Port | Component | Purpose | +|------|-----------|---------| +| 8080 | Gateway | gRPC + HTTP multiplexed (default `DEFAULT_SERVER_PORT`) | +| 2222 | Sandbox | Port mapping in container spec (default `DEFAULT_SSH_PORT`); currently inert -- SSH daemon uses Unix socket only | +| 3128 | Sandbox proxy | HTTP CONNECT proxy (inside container, on inner netns host side) | +| 0 (ephemeral) | Host (via pasta) | Published mapping for container SSH port | + +## Key Source Files + +| File | What it controls | +|------|-----------------| +| `crates/openshell-driver-podman/src/driver.rs` | Bridge network creation, gRPC endpoint auto-detection, rootless checks | +| `crates/openshell-driver-podman/src/container.rs` | Container spec: network mode, port mappings, hostadd, tmpfs, capabilities | +| `crates/openshell-driver-podman/src/client.rs` | Podman REST API calls for network ensure/inspect, port discovery | +| `crates/openshell-driver-podman/src/config.rs` | Network name, socket path, SSH port, gateway port defaults | +| `crates/openshell-sandbox/src/sandbox/linux/netns.rs` | Inner network namespace: veth pair, IP addressing, iptables rules | +| `crates/openshell-sandbox/src/proxy.rs` | HTTP CONNECT proxy: OPA policy, SSRF protection, L7 inspection | +| `crates/openshell-sandbox/src/ssh.rs` | SSH daemon on Unix socket, shell process netns entry via `setns()` | +| `crates/openshell-sandbox/src/supervisor_session.rs` | gRPC ConnectSupervisor stream, RelayStream for SSH tunneling | +| `crates/openshell-sandbox/src/grpc_client.rs` | gRPC channel to gateway (mTLS or plaintext, keep-alive, adaptive windowing) | +| `crates/openshell-server/src/ssh_tunnel.rs` | Gateway-side SSH tunnel: HTTP CONNECT endpoint, relay bridging | +| `crates/openshell-server/src/supervisor_session.rs` | SupervisorSessionRegistry, relay claim/open lifecycle | +| `crates/openshell-server/src/compute/mod.rs` | `ComputeRuntime::new_podman()` -- Podman compute driver initialization | +| `crates/openshell-core/src/config.rs` | Default constants: ports, network name | diff --git a/architecture/sandbox-connect.md b/architecture/sandbox-connect.md index 88505c7f16..f9a0c491e0 100644 --- a/architecture/sandbox-connect.md +++ b/architecture/sandbox-connect.md @@ -8,10 +8,14 @@ Sandbox connect provides secure remote access into running sandbox environments. 2. **Command execution** (`sandbox create -- `) -- runs a command over SSH with stdout/stderr piped back 3. **File sync** (`sandbox create --upload`) -- uploads local files into the sandbox before command execution -Gateway connectivity is **supervisor-initiated**: the gateway never dials the sandbox pod. On startup, each sandbox's supervisor opens a long-lived bidirectional gRPC stream (`ConnectSupervisor`) to the gateway and holds it for the sandbox's lifetime. When a client asks the gateway for SSH, the gateway sends a `RelayOpen` message over that stream; the supervisor responds by initiating a `RelayStream` gRPC call that rides the same TCP+TLS+HTTP/2 connection as a new multiplexed stream. The supervisor bridges the bytes of that stream into a root-owned Unix socket where the embedded SSH daemon listens. +Gateway connectivity is **supervisor-initiated**: the gateway never dials the sandbox pod. On startup, each sandbox's supervisor opens a long-lived bidirectional gRPC stream (`ConnectSupervisor`) to the gateway and holds it for the sandbox's lifetime. **`CreateSshSession` → HTTP CONNECT and `ExecSandbox` both depend on that registration**: `open_relay` blocks until a live `ConnectSupervisor` entry exists for the `sandbox_id`; if the supervisor never registers (wrong endpoint, bad env, crash loop), the client hits the supervisor-session wait timeout instead of getting a relay. When a client asks the gateway for SSH, the gateway sends a `RelayOpen` message over that stream; the supervisor responds by initiating a `RelayStream` gRPC call that rides the same TCP+TLS+HTTP/2 connection as a new multiplexed stream. The supervisor bridges the bytes of that stream into a root-owned Unix socket where the embedded SSH daemon listens. **The in-container sshd is reached only on that local Unix socket** — the supervisor `UnixStream::connect`s to it. Do not assume the relay path terminates at a container-exposed TCP listener for sshd; any optional TCP surface is separate from the gateway relay bridge. There is also a gateway-side `ExecSandbox` gRPC RPC that executes commands inside sandboxes without requiring an external SSH client. It uses the same relay mechanism. +### Podman and relay environment + +The **Podman** compute driver (`crates/openshell-driver-podman/src/container.rs`, `build_env` / `build_container_spec`) must inject the same **relay-critical** environment variables into the container as the Kubernetes driver: `OPENSHELL_ENDPOINT` (gateway gRPC), `OPENSHELL_SANDBOX_ID`, and `OPENSHELL_SSH_SOCKET_PATH` (Unix path the embedded sshd binds and the supervisor dials). Without `OPENSHELL_SSH_SOCKET_PATH`, the in-container `openshell-sandbox` process does not know where to create the socket; without `OPENSHELL_ENDPOINT` / `OPENSHELL_SANDBOX_ID`, the supervisor cannot complete `ConnectSupervisor`, so the gateway never has a session to target with `RelayOpen`. Driver-owned keys overwrite user spec/template env so these cannot be overridden. **Podman container readiness** (libpod `HealthConfig` in `build_container_spec`) treats the sandbox as ready when a sentinel file exists, **or** `test -S` passes on the configured `sandbox_ssh_socket_path` (**supervisor / Unix-socket path**), **or** a legacy TCP listen check on the published SSH port — so the `Ready` phase used by `CreateSshSession` and the SSH tunnel can reflect Unix-socket–based startup, not only a TCP listener. + ## Two-Plane Architecture The supervisor and gateway maintain two logical planes over **one TCP+TLS connection**, multiplexed by HTTP/2 streams: @@ -619,11 +623,11 @@ The sandbox SSH daemon's exit thread waits for the reader thread to finish forwa ### Sandbox environment variables -These are injected into sandbox pods by the Kubernetes driver (`crates/openshell-driver-kubernetes/src/driver.rs`): +These are injected into compute-backed sandboxes by the **Kubernetes** driver (`crates/openshell-driver-kubernetes/src/driver.rs`) and the **Podman** driver (`crates/openshell-driver-podman/src/container.rs`). Together they are required for **persistent `ConnectSupervisor` registration and relay** (see [Podman and relay environment](#podman-and-relay-environment) for the Podman-specific fix): | Variable | Description | |---|---| -| `OPENSHELL_SSH_SOCKET_PATH` | Filesystem path for the embedded SSH server's Unix socket (default `/run/openshell/ssh.sock`) | +| `OPENSHELL_SSH_SOCKET_PATH` | Filesystem path for the embedded SSH server's Unix socket (default `/run/openshell/ssh.sock`); must align with gateway `sandbox_ssh_socket_path` | | `OPENSHELL_ENDPOINT` | Gateway gRPC endpoint; the supervisor uses this to open `ConnectSupervisor` | | `OPENSHELL_SANDBOX_ID` | Identifier reported in `SupervisorHello` | diff --git a/crates/openshell-cli/src/auth.rs b/crates/openshell-cli/src/auth.rs index 6325ebf9c4..e961828c47 100644 --- a/crates/openshell-cli/src/auth.rs +++ b/crates/openshell-cli/src/auth.rs @@ -86,6 +86,20 @@ fn generate_confirmation_code() -> String { /// 4. Waits for the XHR POST callback with the CF JWT and matching code /// 5. Returns the token pub async fn browser_auth_flow(gateway_endpoint: &str) -> Result { + // Short-circuit when the browser is suppressed (CI, e2e tests, headless + // environments). Without this early return the function still binds a TCP + // listener, spawns a callback server, and waits the full AUTH_TIMEOUT + // (120 s) for a POST that will never arrive. + let no_browser = std::env::var("OPENSHELL_NO_BROWSER") + .map(|v| v == "1" || v.eq_ignore_ascii_case("true")) + .unwrap_or(false); + if no_browser { + return Err(miette::miette!( + "authentication skipped (OPENSHELL_NO_BROWSER is set).\n\ + Authenticate later with: openshell gateway login" + )); + } + let listener = TcpListener::bind("127.0.0.1:0").await.into_diagnostic()?; let local_addr = listener.local_addr().into_diagnostic()?; let callback_port = local_addr.port(); @@ -108,37 +122,24 @@ pub async fn browser_auth_flow(gateway_endpoint: &str) -> Result { gateway_endpoint.to_string(), )); - // Allow suppressing the browser popup via environment variable (useful for - // CI, e2e tests, and headless environments). - let no_browser = std::env::var("OPENSHELL_NO_BROWSER") - .map(|v| v == "1" || v.eq_ignore_ascii_case("true")) - .unwrap_or(false); - // Prompt the user before opening the browser. eprintln!(" Confirmation code: {code}"); eprintln!(" Verify this code matches your browser before clicking Connect."); eprintln!(); - if no_browser { - eprintln!("Browser opening suppressed (OPENSHELL_NO_BROWSER is set)."); + eprint!("Press Enter to open the browser for authentication..."); + std::io::stderr().flush().ok(); + let mut _input = String::new(); + std::io::stdin().read_line(&mut _input).ok(); + + if let Err(e) = open_browser(&auth_url) { + debug!(error = %e, "failed to open browser"); + eprintln!("Could not open browser automatically."); eprintln!("Open this URL in your browser:"); eprintln!(" {auth_url}"); eprintln!(); } else { - eprint!("Press Enter to open the browser for authentication..."); - std::io::stderr().flush().ok(); - let mut _input = String::new(); - std::io::stdin().read_line(&mut _input).ok(); - - if let Err(e) = open_browser(&auth_url) { - debug!(error = %e, "failed to open browser"); - eprintln!("Could not open browser automatically."); - eprintln!("Open this URL in your browser:"); - eprintln!(" {auth_url}"); - eprintln!(); - } else { - eprintln!("Browser opened."); - } + eprintln!("Browser opened."); } // Wait for the callback or timeout. diff --git a/crates/openshell-cli/src/main.rs b/crates/openshell-cli/src/main.rs index 6239978d75..9a7c412169 100644 --- a/crates/openshell-cli/src/main.rs +++ b/crates/openshell-cli/src/main.rs @@ -1085,7 +1085,7 @@ enum SandboxCommands { #[command(help_template = LEAF_HELP_TEMPLATE, next_help_heading = "FLAGS")] Create { /// Optional sandbox name (auto-generated when omitted). - #[arg(long)] + #[arg(long, add = ArgValueCompleter::new(completers::complete_sandbox_names))] name: Option, /// Sandbox source: a community sandbox name (e.g., `openclaw`), a path @@ -1198,7 +1198,7 @@ enum SandboxCommands { no_auto_providers: bool, /// Command to run after "--" (defaults to an interactive shell). - #[arg(trailing_var_arg = true)] + #[arg(last = true, allow_hyphen_values = true)] command: Vec, }, @@ -1689,6 +1689,23 @@ async fn main() -> Result<()> { ) .init(); + // Propagate verbosity to the OpenSSH LogLevel used by SSH subprocesses. + // Only set the env var when it hasn't been explicitly overridden by the + // user, so `OPENSHELL_SSH_LOG_LEVEL=DEBUG openshell ...` still wins. + if std::env::var("OPENSHELL_SSH_LOG_LEVEL").is_err() { + let ssh_log_level = match cli.verbose { + 0 => "ERROR", + 1 => "INFO", + _ => "DEBUG", + }; + // SAFETY: Called early in main() before spawning async tasks that + // read the environment, so no concurrent readers exist. + #[allow(unsafe_code)] + unsafe { + std::env::set_var("OPENSHELL_SSH_LOG_LEVEL", ssh_log_level); + } + } + match cli.command { // ----------------------------------------------------------- // Gateway commands (was `cluster` / `cluster admin`) @@ -3358,6 +3375,58 @@ mod tests { } } + // ── sandbox create arg-shape tests ─────────────────────────────────── + + /// Verify that `sandbox create --name ` still parses as a named + /// option rather than a positional argument. The `ArgValueCompleter` + /// attribute must be combined with `long` on the `name` field; without + /// `long`, clap treats the field as positional and `--name` is rejected. + #[test] + fn sandbox_create_name_is_a_named_flag() { + // Build the CLI app and try to parse `sandbox create --name foo`. + // `try_parse_from` returns Err if clap rejects the arguments. + let result = Cli::try_parse_from(["openshell", "sandbox", "create", "--name", "my-sb"]); + assert!( + result.is_ok(), + "sandbox create --name should parse as a named flag: {:?}", + result.err() + ); + if let Ok(cli) = result { + if let Some(Commands::Sandbox { + command: Some(SandboxCommands::Create { name, .. }), + .. + }) = cli.command + { + assert_eq!(name.as_deref(), Some("my-sb")); + } else { + panic!("expected SandboxCommands::Create"); + } + } + } + + /// Verify that `sandbox create` without `--name` still works (name is + /// optional and auto-generated when omitted). + #[test] + fn sandbox_create_name_is_optional() { + let result = Cli::try_parse_from(["openshell", "sandbox", "create"]); + assert!( + result.is_ok(), + "sandbox create without --name should be accepted: {:?}", + result.err() + ); + if let Ok(cli) = result { + if let Some(Commands::Sandbox { + command: Some(SandboxCommands::Create { name, .. }), + .. + }) = cli.command + { + assert_eq!(name, None); + } else { + panic!("expected SandboxCommands::Create"); + } + } + } + /// Ensure every provider registered in `ProviderRegistry` has a /// corresponding `CliProviderType` variant (and vice-versa). /// This test would have caught the missing `Copilot` variant from #707. diff --git a/crates/openshell-cli/src/ssh.rs b/crates/openshell-cli/src/ssh.rs index 54321bfccd..f660754281 100644 --- a/crates/openshell-cli/src/ssh.rs +++ b/crates/openshell-cli/src/ssh.rs @@ -134,6 +134,11 @@ async fn ssh_session_config( } fn ssh_base_command(proxy_command: &str) -> Command { + // SSH log level follows the program's verbosity. main() maps the `-v` + // count to OPENSHELL_SSH_LOG_LEVEL; an explicit env-var override wins. + let ssh_log_level = + std::env::var("OPENSHELL_SSH_LOG_LEVEL").unwrap_or_else(|_| "ERROR".to_string()); + let mut command = Command::new("ssh"); command .arg("-o") @@ -145,7 +150,7 @@ fn ssh_base_command(proxy_command: &str) -> Command { .arg("-o") .arg("GlobalKnownHostsFile=/dev/null") .arg("-o") - .arg("LogLevel=ERROR") + .arg(format!("LogLevel={ssh_log_level}")) // Detect a dead relay within ~45s. The relay rides on a TCP connection // that the client has no way to observe silently dropping (gateway // restart, supervisor restart, cluster failover), so fall back to diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index d9440745d3..2fbdb1b1d5 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -9,6 +9,39 @@ use std::net::SocketAddr; use std::path::PathBuf; use std::str::FromStr; +// ── Public default constants ──────────────────────────────────────────── +// +// Canonical source for default values used across multiple crates. +// Clap `default_value_t` annotations and runtime fallbacks should +// reference these constants instead of hardcoding literals. + +/// Default SSH port inside sandbox containers. +pub const DEFAULT_SSH_PORT: u16 = 2222; + +/// Default server / SSH gateway port. +pub const DEFAULT_SERVER_PORT: u16 = 8080; + +/// Default container stop timeout in seconds (SIGTERM → SIGKILL). +pub const DEFAULT_STOP_TIMEOUT_SECS: u32 = 10; + +/// Default allowed clock skew for SSH handshake validation, in seconds. +pub const DEFAULT_SSH_HANDSHAKE_SKEW_SECS: u64 = 300; + +/// Default Podman bridge network name. +pub const DEFAULT_NETWORK_NAME: &str = "openshell"; + +/// Default OCI image for the openshell-sandbox supervisor binary. +pub const DEFAULT_SUPERVISOR_IMAGE: &str = "openshell/supervisor:latest"; + +/// Default image pull policy for sandbox images. +pub const DEFAULT_IMAGE_PULL_POLICY: &str = "missing"; + +/// Default Kubernetes namespace for sandbox resources. +pub const DEFAULT_K8S_NAMESPACE: &str = "openshell"; + +/// CDI device identifier for requesting all NVIDIA GPUs. +pub const CDI_GPU_DEVICE_ALL: &str = "nvidia.com/gpu=all"; + /// Compute backends the gateway can orchestrate sandboxes through. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -92,7 +125,7 @@ pub struct Config { pub sandbox_namespace: String, /// Default container image for sandboxes. - #[serde(default)] + #[serde(default = "default_sandbox_image")] pub sandbox_image: String, /// Kubernetes `imagePullPolicy` for sandbox pods (e.g. `Always`, @@ -119,6 +152,10 @@ pub struct Config { #[serde(default = "default_ssh_connect_path")] pub ssh_connect_path: String, + /// SSH listen port inside sandbox containers that expose a TCP endpoint. + #[serde(default = "default_sandbox_ssh_port")] + pub sandbox_ssh_port: u16, + /// Filesystem path where the sandbox supervisor binds its SSH Unix /// socket. The supervisor is passed this path via /// `OPENSHELL_SSH_SOCKET_PATH` / `--ssh-socket-path` and connects its @@ -195,12 +232,13 @@ impl Config { database_url: String::new(), compute_drivers: default_compute_drivers(), sandbox_namespace: default_sandbox_namespace(), - sandbox_image: String::new(), + sandbox_image: default_sandbox_image(), sandbox_image_pull_policy: String::new(), grpc_endpoint: String::new(), ssh_gateway_host: default_ssh_gateway_host(), ssh_gateway_port: default_ssh_gateway_port(), ssh_connect_path: default_ssh_connect_path(), + sandbox_ssh_port: default_sandbox_ssh_port(), sandbox_ssh_socket_path: default_sandbox_ssh_socket_path(), ssh_handshake_secret: String::new(), ssh_handshake_skew_secs: default_ssh_handshake_skew_secs(), @@ -302,6 +340,13 @@ impl Config { self } + /// Create a new configuration with the sandbox SSH port. + #[must_use] + pub const fn with_sandbox_ssh_port(mut self, port: u16) -> Self { + self.sandbox_ssh_port = port; + self + } + /// Create a new configuration with the SSH handshake secret. #[must_use] pub fn with_ssh_handshake_secret(mut self, secret: impl Into) -> Self { @@ -350,6 +395,10 @@ fn default_sandbox_namespace() -> String { "default".to_string() } +fn default_sandbox_image() -> String { + format!("{}/base:latest", crate::image::DEFAULT_COMMUNITY_REGISTRY) +} + fn default_compute_drivers() -> Vec { vec![ComputeDriverKind::Kubernetes] } @@ -359,7 +408,7 @@ fn default_ssh_gateway_host() -> String { } const fn default_ssh_gateway_port() -> u16 { - 8080 + DEFAULT_SERVER_PORT } fn default_ssh_connect_path() -> String { @@ -370,8 +419,12 @@ fn default_sandbox_ssh_socket_path() -> String { "/run/openshell/ssh.sock".to_string() } +const fn default_sandbox_ssh_port() -> u16 { + DEFAULT_SSH_PORT +} + const fn default_ssh_handshake_skew_secs() -> u64 { - 300 + DEFAULT_SSH_HANDSHAKE_SKEW_SECS } const fn default_ssh_session_ttl_secs() -> u64 { diff --git a/crates/openshell-core/src/error.rs b/crates/openshell-core/src/error.rs index 2399368fc8..9d16501355 100644 --- a/crates/openshell-core/src/error.rs +++ b/crates/openshell-core/src/error.rs @@ -103,3 +103,20 @@ impl Error { } } } + +/// Error type shared by all compute driver implementations. +/// +/// Both the Podman and Kubernetes drivers map their backend-specific +/// errors into these variants before crossing crate boundaries. +#[derive(Debug, Error)] +pub enum ComputeDriverError { + /// The requested sandbox already exists. + #[error("sandbox already exists")] + AlreadyExists, + /// A precondition for the operation was not met. + #[error("{0}")] + Precondition(String), + /// Generic error message. + #[error("{0}")] + Message(String), +} diff --git a/crates/openshell-core/src/lib.rs b/crates/openshell-core/src/lib.rs index c0b08f1a57..28afdd4143 100644 --- a/crates/openshell-core/src/lib.rs +++ b/crates/openshell-core/src/lib.rs @@ -20,7 +20,7 @@ pub mod proto; pub mod settings; pub use config::{ComputeDriverKind, Config, TlsConfig}; -pub use error::{Error, Result}; +pub use error::{ComputeDriverError, Error, Result}; /// Build version string derived from git metadata. /// diff --git a/crates/openshell-driver-podman/Cargo.toml b/crates/openshell-driver-podman/Cargo.toml new file mode 100644 index 0000000000..51ac698de0 --- /dev/null +++ b/crates/openshell-driver-podman/Cargo.toml @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "openshell-driver-podman" +description = "Podman compute driver for OpenShell" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[[bin]] +name = "openshell-driver-podman" +path = "src/main.rs" + +[dependencies] +openshell-core = { path = "../openshell-core" } + +tokio = { workspace = true } +tonic = { workspace = true, features = ["transport"] } +futures = { workspace = true } +tokio-stream = { workspace = true } +hyper = { workspace = true } +hyper-util = { workspace = true } +http-body-util = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +clap = { workspace = true } +nix = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } +thiserror = { workspace = true } +miette = { workspace = true } + +[dev-dependencies] +temp-env = "0.3" + +[lints] +workspace = true diff --git a/crates/openshell-driver-podman/src/client.rs b/crates/openshell-driver-podman/src/client.rs new file mode 100644 index 0000000000..1c590db2c1 --- /dev/null +++ b/crates/openshell-driver-podman/src/client.rs @@ -0,0 +1,829 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Thin async HTTP client for the Podman REST API over a Unix socket. + +use http_body_util::{BodyExt, Full}; +use hyper::Request; +use hyper::body::Bytes; +use hyper_util::rt::TokioIo; +use serde::de::DeserializeOwned; +use serde_json::Value; +use std::collections::HashMap; +use std::path::PathBuf; +use std::pin::Pin; +use std::time::Duration; +use tokio::net::UnixStream; +use tokio::sync::mpsc; +use tracing::debug; + +/// Podman libpod API version prefix. +const API_VERSION: &str = "v5.0.0"; + +/// Timeout for individual Podman API calls. +const API_TIMEOUT: Duration = Duration::from_secs(30); + +/// Maximum allowed size for the event stream line buffer (1 MB). +const MAX_EVENT_BUFFER: usize = 1_048_576; + +#[derive(Debug, thiserror::Error)] +pub enum PodmanApiError { + #[error("podman API not found (404): {0}")] + NotFound(String), + #[error("podman API conflict (409): {0}")] + Conflict(String), + #[error("podman API error ({status}): {message}")] + Api { status: u16, message: String }, + #[error("connection error: {0}")] + Connection(String), + #[error("timeout after {0:?}")] + Timeout(Duration), + #[error("JSON error: {0}")] + Json(String), + #[error("invalid input: {0}")] + InvalidInput(String), +} + +/// Maximum resource name length. Podman container names become directory +/// names in the storage driver, so we cap at 255 to stay within ext4/xfs +/// filename limits. +const MAX_NAME_LEN: usize = 255; + +/// Validate that a resource name is safe for URL path interpolation. +/// +/// Valid names start with an alphanumeric character and contain only +/// alphanumerics, dots, underscores, and hyphens — matching Podman's +/// own naming rules. Names longer than [`MAX_NAME_LEN`] are rejected. +pub(crate) fn validate_name(name: &str) -> Result<(), PodmanApiError> { + // Regex-equivalent: ^[a-zA-Z0-9][a-zA-Z0-9._-]*$ + if name.is_empty() { + return Err(PodmanApiError::InvalidInput( + "name must not be empty".to_string(), + )); + } + if name.len() > MAX_NAME_LEN { + return Err(PodmanApiError::InvalidInput(format!( + "name exceeds maximum length of {MAX_NAME_LEN} characters (got {})", + name.len() + ))); + } + let bytes = name.as_bytes(); + if !bytes[0].is_ascii_alphanumeric() { + return Err(PodmanApiError::InvalidInput(format!( + "name must start with an alphanumeric character: {name:?}" + ))); + } + if !bytes + .iter() + .all(|&b| b.is_ascii_alphanumeric() || b == b'.' || b == b'_' || b == b'-') + { + return Err(PodmanApiError::InvalidInput(format!( + "name contains invalid characters: {name:?}" + ))); + } + Ok(()) +} + +/// A container state snapshot returned by inspect APIs. +#[derive(Debug, Clone, serde::Deserialize)] +#[serde(rename_all = "PascalCase")] +pub struct ContainerInspect { + pub id: String, + pub name: String, + pub state: ContainerState, + #[serde(default)] + pub network_settings: NetworkSettings, + #[serde(default)] + pub config: ContainerConfig, +} + +#[derive(Debug, Clone, serde::Deserialize)] +#[serde(rename_all = "PascalCase")] +pub struct ContainerState { + pub status: String, + pub running: bool, + #[serde(default)] + pub exit_code: i64, + #[serde(rename = "OOMKilled")] + #[serde(default)] + pub oom_killed: bool, + #[serde(default)] + pub health: Option, + #[serde(default)] + pub started_at: Option, + #[serde(default)] + pub finished_at: Option, +} + +#[derive(Debug, Clone, serde::Deserialize)] +#[serde(rename_all = "PascalCase")] +pub struct HealthState { + pub status: String, +} + +#[derive(Debug, Clone, Default, serde::Deserialize)] +#[serde(rename_all = "PascalCase")] +pub struct NetworkSettings { + #[serde(default)] + pub networks: HashMap, + #[serde(default)] + pub ports: HashMap>>, +} + +#[derive(Debug, Clone, serde::Deserialize)] +#[serde(rename_all = "PascalCase")] +pub struct NetworkInfo { + #[serde(rename = "IPAddress")] + #[serde(default)] + pub ip_address: String, +} + +#[derive(Debug, Clone, serde::Deserialize)] +#[serde(rename_all = "PascalCase")] +pub struct PortBinding { + #[serde(default)] + pub host_port: String, + #[serde(rename = "HostIp")] + #[serde(default)] + pub host_ip: String, +} + +#[derive(Debug, Clone, Default, serde::Deserialize)] +#[serde(rename_all = "PascalCase")] +pub struct ContainerConfig { + #[serde(default)] + pub labels: HashMap, +} + +/// A container summary returned by the list API. +#[derive(Debug, Clone, serde::Deserialize)] +#[serde(rename_all = "PascalCase")] +pub struct ContainerListEntry { + pub id: String, + #[serde(default)] + pub names: Vec, + pub state: String, + #[serde(default)] + pub labels: HashMap, + #[serde(default)] + pub ports: Option>, + #[serde(default)] + pub networks: Option>, + #[serde(default)] + pub exit_code: i64, +} + +#[derive(Debug, Clone, serde::Deserialize)] +pub struct PortMappingEntry { + pub host_port: u16, + pub container_port: u16, + pub protocol: String, + #[serde(default)] + pub host_ip: String, +} + +/// A Podman event from the events stream. +#[derive(Debug, Clone, serde::Deserialize)] +#[serde(rename_all = "PascalCase")] +pub struct PodmanEvent { + #[serde(rename = "Type")] + pub event_type: String, + pub action: String, + #[serde(default)] + pub actor: EventActor, + #[serde(rename = "timeNano", default)] + pub time_nano: i64, +} + +#[derive(Debug, Clone, Default, serde::Deserialize)] +#[serde(rename_all = "PascalCase")] +pub struct EventActor { + #[serde(rename = "ID")] + #[serde(default)] + pub id: String, + #[serde(default)] + pub attributes: HashMap, +} + +/// System info response (subset of fields we care about). +#[derive(Debug, Clone, serde::Deserialize)] +pub struct SystemInfo { + pub host: HostInfo, +} + +#[derive(Debug, Clone, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct HostInfo { + #[serde(default)] + pub cgroup_version: String, + #[serde(default)] + pub network_backend: String, +} + +// ── Client ─────────────────────────────────────────────────────────────── + +/// Async Podman REST API client communicating over a Unix socket. +#[derive(Debug, Clone)] +pub struct PodmanClient { + socket_path: PathBuf, +} + +impl PodmanClient { + /// Create a new client targeting the given socket path. + #[must_use] + pub fn new(socket_path: PathBuf) -> Self { + Self { socket_path } + } + + /// Open a new HTTP/1.1 connection to the Podman socket. + async fn connect( + &self, + ) -> Result>, PodmanApiError> { + let stream = UnixStream::connect(&self.socket_path).await.map_err(|e| { + PodmanApiError::Connection(format!("{}: {e}", self.socket_path.display())) + })?; + + let (sender, conn) = hyper::client::conn::http1::handshake(TokioIo::new(stream)) + .await + .map_err(|e| PodmanApiError::Connection(e.to_string()))?; + + tokio::spawn(async move { + if let Err(e) = conn.await { + debug!(error = %e, "Podman API connection closed"); + } + }); + + Ok(sender) + } + + // ── Request infrastructure ─────────────────────────────────────────── + + /// Build an HTTP request from components. + fn build_request( + method: hyper::Method, + path: &str, + body: Full, + content_type: Option<&str>, + ) -> Request> { + let mut builder = Request::builder() + .method(method) + .uri(format!("http://localhost{path}")) + .header("Host", "localhost"); + if let Some(ct) = content_type { + builder = builder.header("Content-Type", ct); + } + builder.body(body).expect("valid request") + } + + /// Send a pre-built HTTP request and return status + body bytes. + async fn send_request( + &self, + req: Request>, + timeout: Duration, + ) -> Result<(hyper::StatusCode, Bytes), PodmanApiError> { + let mut sender = self.connect().await?; + let response = tokio::time::timeout(timeout, sender.send_request(req)) + .await + .map_err(|_| PodmanApiError::Timeout(timeout))? + .map_err(|e| PodmanApiError::Connection(e.to_string()))?; + let status = response.status(); + let bytes = tokio::time::timeout(timeout, response.into_body().collect()) + .await + .map_err(|_| PodmanApiError::Timeout(timeout))? + .map_err(|e| PodmanApiError::Connection(e.to_string()))? + .to_bytes(); + Ok((status, bytes)) + } + + /// Perform a versioned HTTP request and return status + body bytes. + async fn request( + &self, + method: hyper::Method, + path: &str, + body: Option<&Value>, + timeout: Duration, + ) -> Result<(hyper::StatusCode, Bytes), PodmanApiError> { + let (full_body, content_type) = match body { + Some(json) => { + let payload = + serde_json::to_vec(json).map_err(|e| PodmanApiError::Json(e.to_string()))?; + (Full::new(Bytes::from(payload)), Some("application/json")) + } + None => (Full::new(Bytes::new()), None), + }; + let req = Self::build_request( + method, + &format!("/{API_VERSION}{path}"), + full_body, + content_type, + ); + self.send_request(req, timeout).await + } + + /// Perform a request and deserialize the JSON response. + async fn request_json( + &self, + method: hyper::Method, + path: &str, + body: Option<&Value>, + ) -> Result { + let (status, bytes) = self.request(method, path, body, API_TIMEOUT).await?; + if status.is_success() { + serde_json::from_slice(&bytes).map_err(|e| { + PodmanApiError::Json(format!("{e}: {}", String::from_utf8_lossy(&bytes))) + }) + } else { + Err(error_from_response(status.as_u16(), &bytes)) + } + } + + /// Perform a request that returns no meaningful body. + async fn request_ok( + &self, + method: hyper::Method, + path: &str, + body: Option<&Value>, + ) -> Result<(), PodmanApiError> { + let (status, bytes) = self.request(method, path, body, API_TIMEOUT).await?; + let code = status.as_u16(); + if status.is_success() || code == 304 { + Ok(()) + } else { + Err(error_from_response(code, &bytes)) + } + } + + /// Perform a versioned HTTP request with a raw byte body (not JSON). + async fn request_raw( + &self, + method: hyper::Method, + path: &str, + content_type: &str, + body: Bytes, + ) -> Result<(hyper::StatusCode, Bytes), PodmanApiError> { + let req = Self::build_request( + method, + &format!("/{API_VERSION}{path}"), + Full::new(body), + Some(content_type), + ); + self.send_request(req, API_TIMEOUT).await + } + + /// POST a JSON body and ignore 409 Conflict (resource already exists). + async fn create_ignore_conflict(&self, path: &str, body: &Value) -> Result<(), PodmanApiError> { + match self + .request_json::(hyper::Method::POST, path, Some(body)) + .await + { + Ok(_) | Err(PodmanApiError::Conflict(_)) => Ok(()), + Err(e) => Err(e), + } + } + + // ── Container operations ───────────────────────────────────────────── + + /// Create a container from a JSON spec. + pub async fn create_container(&self, spec: &Value) -> Result { + self.request_json(hyper::Method::POST, "/libpod/containers/create", Some(spec)) + .await + } + + /// Start a container by name or ID. + pub async fn start_container(&self, name: &str) -> Result<(), PodmanApiError> { + validate_name(name)?; + self.request_ok( + hyper::Method::POST, + &format!("/libpod/containers/{name}/start"), + None, + ) + .await + } + + /// Stop a container with a grace period in seconds. + pub async fn stop_container( + &self, + name: &str, + timeout_secs: u32, + ) -> Result<(), PodmanApiError> { + validate_name(name)?; + let http_timeout = Duration::from_secs(timeout_secs as u64 + 5); + let (status, bytes) = self + .request( + hyper::Method::POST, + &format!("/libpod/containers/{name}/stop?timeout={timeout_secs}"), + None, + http_timeout, + ) + .await?; + let code = status.as_u16(); + if status.is_success() || code == 304 { + Ok(()) + } else { + Err(error_from_response(code, &bytes)) + } + } + + /// Force-remove a container and its anonymous volumes. + pub async fn remove_container(&self, name: &str) -> Result<(), PodmanApiError> { + validate_name(name)?; + self.request_ok( + hyper::Method::DELETE, + &format!("/libpod/containers/{name}?force=true&v=true"), + None, + ) + .await + } + + /// Inspect a container by name or ID. + pub async fn inspect_container(&self, name: &str) -> Result { + validate_name(name)?; + self.request_json( + hyper::Method::GET, + &format!("/libpod/containers/{name}/json"), + None, + ) + .await + } + + /// List containers matching a label filter (e.g. `"openshell.managed=true"`). + pub async fn list_containers( + &self, + label_filter: &str, + ) -> Result, PodmanApiError> { + let filters = serde_json::json!({"label": [label_filter]}); + let encoded = url_encode(&filters.to_string()); + self.request_json( + hyper::Method::GET, + &format!("/libpod/containers/json?all=true&filters={encoded}"), + None, + ) + .await + } + + // ── Volume operations ──────────────────────────────────────────────── + + /// Create a named volume. Idempotent (conflict is ignored). + pub async fn create_volume(&self, name: &str) -> Result<(), PodmanApiError> { + validate_name(name)?; + self.create_ignore_conflict("/libpod/volumes/create", &serde_json::json!({"Name": name})) + .await + } + + /// Remove a named volume. Idempotent (not-found is ignored). + pub async fn remove_volume(&self, name: &str) -> Result<(), PodmanApiError> { + validate_name(name)?; + match self + .request_ok( + hyper::Method::DELETE, + &format!("/libpod/volumes/{name}"), + None, + ) + .await + { + Ok(()) | Err(PodmanApiError::NotFound(_)) => Ok(()), + Err(e) => Err(e), + } + } + + // ── Network operations ─────────────────────────────────────────────── + + /// Create a bridge network with DNS enabled. Idempotent. + pub async fn ensure_network(&self, name: &str) -> Result<(), PodmanApiError> { + validate_name(name)?; + self.create_ignore_conflict( + "/libpod/networks/create", + &serde_json::json!({ + "name": name, + "driver": "bridge", + "dns_enabled": true, + }), + ) + .await + } + + /// Inspect a network and return the gateway IP of its first subnet. + /// + /// The gateway IP is the host's address on the bridge network, used by + /// sandbox containers to call back to the gateway server. + pub async fn network_gateway_ip(&self, name: &str) -> Result, PodmanApiError> { + validate_name(name)?; + let encoded = url_encode(name); + let path = format!("/libpod/networks/{encoded}/json"); + let resp: Value = self.request_json(hyper::Method::GET, &path, None).await?; + // The response has "subnets": [{"gateway": "10.89.1.1", "subnet": "..."}] + let gateway = resp + .get("subnets") + .and_then(|s| s.as_array()) + .and_then(|arr| arr.first()) + .and_then(|sub| sub.get("gateway")) + .and_then(|g| g.as_str()) + .map(String::from); + Ok(gateway) + } + + // ── Secret operations ──────────────────────────────────────────────── + + /// Create a Podman secret with the given name and raw value. + /// + /// Idempotent: if a secret with the same name already exists it is + /// replaced (delete + recreate) so the value is always up-to-date. + pub async fn create_secret(&self, name: &str, value: &[u8]) -> Result<(), PodmanApiError> { + validate_name(name)?; + let encoded_name = url_encode(name); + let path = format!("/libpod/secrets/create?name={encoded_name}"); + let (status, bytes) = self + .request_raw( + hyper::Method::POST, + &path, + "application/octet-stream", + Bytes::copy_from_slice(value), + ) + .await?; + + match status.as_u16() { + 200 | 201 => Ok(()), + 409 => { + // Secret already exists — replace it. + self.remove_secret(name).await?; + let (status2, bytes2) = self + .request_raw( + hyper::Method::POST, + &path, + "application/octet-stream", + Bytes::copy_from_slice(value), + ) + .await?; + if status2.is_success() { + Ok(()) + } else { + Err(error_from_response(status2.as_u16(), &bytes2)) + } + } + _ => Err(error_from_response(status.as_u16(), &bytes)), + } + } + + /// Remove a Podman secret by name. Idempotent (not-found is ignored). + pub async fn remove_secret(&self, name: &str) -> Result<(), PodmanApiError> { + validate_name(name)?; + match self + .request_ok( + hyper::Method::DELETE, + &format!("/libpod/secrets/{name}"), + None, + ) + .await + { + Ok(()) | Err(PodmanApiError::NotFound(_)) => Ok(()), + Err(e) => Err(e), + } + } + + // ── Image operations ──────────────────────────────────────────────── + + /// Pull an image if it is not already present locally. + /// + /// Uses the `policy` parameter to decide whether to pull: + /// - `"always"` — always pull, even if a local copy exists + /// - `"missing"` — pull only when no local copy exists (default) + /// - `"never"` — never pull, fail if not local + /// - `"newer"` — pull only if the remote image is newer + /// + /// The pull `policy` is passed directly to Podman's API so that + /// Podman handles local-image resolution and registry fallback + /// natively. This avoids name-resolution mismatches between the + /// exists API and the local image store (e.g. `openshell/supervisor:dev` + /// vs `localhost/openshell/supervisor:dev`). + /// + /// The Podman pull endpoint streams NDJSON progress. We consume the + /// entire stream and check for an `error` field in the final object. + pub async fn pull_image(&self, reference: &str, policy: &str) -> Result<(), PodmanApiError> { + let path = format!( + "/libpod/images/pull?reference={}&policy={}", + url_encode(reference), + url_encode(policy), + ); + // Image pulls can be slow — use a generous timeout. + let pull_timeout = Duration::from_secs(600); + let (status, bytes) = self + .request(hyper::Method::POST, &path, None, pull_timeout) + .await?; + if !status.is_success() { + return Err(error_from_response(status.as_u16(), &bytes)); + } + // The response is NDJSON. Check the last line for an error field. + let body = String::from_utf8_lossy(&bytes); + if let Some(last_line) = body.lines().filter(|l| !l.is_empty()).last() { + if let Ok(obj) = serde_json::from_str::(last_line) { + if let Some(err) = obj.get("error").and_then(|v| v.as_str()) { + if !err.is_empty() { + return Err(PodmanApiError::Api { + status: 500, + message: format!("image pull failed: {err}"), + }); + } + } + } + } + Ok(()) + } + + // ── System operations ──────────────────────────────────────────────── + + /// Ping the Podman API to verify connectivity. + pub async fn ping(&self) -> Result<(), PodmanApiError> { + // _ping is outside the versioned API path. + let req = Self::build_request(hyper::Method::GET, "/_ping", Full::new(Bytes::new()), None); + let (status, _) = self.send_request(req, API_TIMEOUT).await?; + if status.is_success() { + Ok(()) + } else { + Err(PodmanApiError::Api { + status: status.as_u16(), + message: "ping failed".to_string(), + }) + } + } + + /// Get system info. + pub async fn system_info(&self) -> Result { + self.request_json(hyper::Method::GET, "/libpod/info", None) + .await + } + + // ── Event streaming ────────────────────────────────────────────────── + + /// Start streaming container events filtered by label. + /// + /// Events are sent to the returned receiver. The background task runs + /// until the receiver is dropped. + pub async fn events_stream( + &self, + label_filter: &str, + ) -> Result>, PodmanApiError> { + let filters = serde_json::json!({ + "label": [label_filter], + "type": ["container"], + }); + let encoded = url_encode(&filters.to_string()); + let path = + format!("http://localhost/{API_VERSION}/libpod/events?stream=true&filters={encoded}"); + + let mut sender = self.connect().await?; + + let req = Request::builder() + .method(hyper::Method::GET) + .uri(&path) + .header("Host", "localhost") + .body(Full::new(Bytes::new())) + .map_err(|e| PodmanApiError::Connection(e.to_string()))?; + + let response = tokio::time::timeout(API_TIMEOUT, sender.send_request(req)) + .await + .map_err(|_| PodmanApiError::Timeout(API_TIMEOUT))? + .map_err(|e| PodmanApiError::Connection(e.to_string()))?; + + if !response.status().is_success() { + return Err(PodmanApiError::Api { + status: response.status().as_u16(), + message: "events stream request failed".to_string(), + }); + } + + let (tx, rx) = mpsc::channel(256); + let body = response.into_body(); + + tokio::spawn(async move { + let mut buffer = Vec::new(); + let mut body = body; + + loop { + use hyper::body::Body; + + let frame = + match std::future::poll_fn(|cx| Pin::new(&mut body).poll_frame(cx)).await { + Some(Ok(frame)) => frame, + Some(Err(e)) => { + let _ = tx + .send(Err(PodmanApiError::Connection(e.to_string()))) + .await; + break; + } + None => break, + }; + + if let Some(data) = frame.data_ref() { + buffer.extend_from_slice(data); + } + + if buffer.len() > MAX_EVENT_BUFFER { + tracing::error!("event stream buffer exceeded maximum size, disconnecting"); + let _ = tx + .send(Err(PodmanApiError::Connection( + "event buffer exceeded 1 MB limit".to_string(), + ))) + .await; + break; + } + + // Parse complete newline-delimited JSON lines. + while let Some(pos) = buffer.iter().position(|&b| b == b'\n') { + let line: Vec = buffer.drain(..=pos).collect(); + let trimmed = line.strip_suffix(&[b'\n']).unwrap_or(&line); + if trimmed.is_empty() { + continue; + } + let event = serde_json::from_slice::(trimmed).map_err(|e| { + PodmanApiError::Json(format!("{e}: {}", String::from_utf8_lossy(trimmed))) + }); + if tx.send(event).await.is_err() { + return; + } + } + } + }); + + Ok(rx) + } +} + +// ── Helpers ────────────────────────────────────────────────────────────── + +fn error_from_response(status: u16, bytes: &Bytes) -> PodmanApiError { + let message = serde_json::from_slice::(bytes) + .ok() + .and_then(|v| { + v.get("message") + .or_else(|| v.get("cause")) + .and_then(Value::as_str) + .map(String::from) + }) + .unwrap_or_else(|| String::from_utf8_lossy(bytes).to_string()); + + match status { + 404 => PodmanApiError::NotFound(message), + 409 => PodmanApiError::Conflict(message), + _ => PodmanApiError::Api { status, message }, + } +} + +/// Minimal percent-encoding for query parameter values. +/// +/// Note: `percent-encoding` is available as a transitive dependency but is not +/// a direct dependency of this crate. Rather than adding a new dep for one +/// call site, we keep this self-contained implementation. +fn url_encode(s: &str) -> String { + s.bytes() + .map(|b| match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + String::from(b as char) + } + _ => format!("%{b:02X}"), + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn url_encode_encodes_special_characters() { + assert_eq!(url_encode("hello world"), "hello%20world"); + assert_eq!(url_encode("a=b&c=d"), "a%3Db%26c%3Dd"); + assert_eq!(url_encode("safe-_.~chars"), "safe-_.~chars"); + } + + #[test] + fn validate_name_accepts_valid_names() { + // alphanumeric, dots, hyphens, underscores + assert!(validate_name("my-container").is_ok()); + assert!(validate_name("my_container.v2").is_ok()); + assert!(validate_name("a").is_ok()); + assert!(validate_name("Container123").is_ok()); + } + + #[test] + fn validate_name_rejects_invalid_names() { + assert!(validate_name("").is_err()); // empty + assert!(validate_name("-leading").is_err()); // starts with dash + assert!(validate_name(".leading").is_err()); // starts with dot + assert!(validate_name("has/slash").is_err()); // path traversal + assert!(validate_name("../etc").is_err()); // path traversal + assert!(validate_name("has space").is_err()); // space + assert!(validate_name("has%20encoded").is_err()); // percent + assert!(validate_name("has?query").is_err()); // query char + } + + #[test] + fn validate_name_rejects_names_exceeding_max_length() { + let long_name = format!("a{}", "b".repeat(MAX_NAME_LEN)); + assert!(long_name.len() > MAX_NAME_LEN); + assert!(validate_name(&long_name).is_err()); + + // Exactly at the limit should be accepted. + let exact_name = "a".repeat(MAX_NAME_LEN); + assert!(validate_name(&exact_name).is_ok()); + } +} diff --git a/crates/openshell-driver-podman/src/config.rs b/crates/openshell-driver-podman/src/config.rs new file mode 100644 index 0000000000..8eb2d97410 --- /dev/null +++ b/crates/openshell-driver-podman/src/config.rs @@ -0,0 +1,199 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use openshell_core::config::{ + DEFAULT_NETWORK_NAME, DEFAULT_SSH_HANDSHAKE_SKEW_SECS, DEFAULT_SSH_PORT, + DEFAULT_STOP_TIMEOUT_SECS, DEFAULT_SUPERVISOR_IMAGE, +}; +use std::path::PathBuf; +use std::str::FromStr; + +/// Image pull policy for sandbox and supervisor images. +/// +/// Controls when the Podman driver fetches a newer copy of an OCI image +/// from the registry. +#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ImagePullPolicy { + /// Always pull, even if a local copy exists. + Always, + /// Pull only when no local copy exists (default). + #[default] + Missing, + /// Never pull; fail if not available locally. + Never, + /// Pull only if the remote image is newer. + Newer, +} + +impl ImagePullPolicy { + /// Return the policy string expected by the Podman libpod API. + #[must_use] + pub fn as_str(&self) -> &'static str { + match self { + Self::Always => "always", + Self::Missing => "missing", + Self::Never => "never", + Self::Newer => "newer", + } + } +} + +impl std::fmt::Display for ImagePullPolicy { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +impl FromStr for ImagePullPolicy { + type Err = String; + + fn from_str(s: &str) -> Result { + match s.to_ascii_lowercase().as_str() { + "always" => Ok(Self::Always), + "missing" => Ok(Self::Missing), + "never" => Ok(Self::Never), + "newer" => Ok(Self::Newer), + other => Err(format!( + "invalid pull policy '{other}'; expected one of: always, missing, never, newer" + )), + } + } +} + +#[derive(Clone)] +pub struct PodmanComputeConfig { + /// Path to the Podman API Unix socket. + /// Default: `$XDG_RUNTIME_DIR/podman/podman.sock` + pub socket_path: PathBuf, + /// Default OCI image for sandboxes. + pub default_image: String, + /// Image pull policy for sandbox images. + pub image_pull_policy: ImagePullPolicy, + /// Gateway gRPC endpoint the sandbox connects back to. + /// + /// When empty, the driver auto-detects the endpoint using + /// `gateway_port` and `host.containers.internal`. + pub grpc_endpoint: String, + /// Port the gateway server is actually listening on. + /// + /// Used by the driver's auto-detection fallback when `grpc_endpoint` + /// is empty. The server must set this to `config.bind_address.port()` + /// so the correct port is used even when `--port` differs from the + /// default. Defaults to [`openshell_core::config::DEFAULT_SERVER_PORT`]. + pub gateway_port: u16, + /// Unix socket path the in-container supervisor bridges relay traffic to. + pub sandbox_ssh_socket_path: String, + /// Name of the Podman bridge network. + /// Created automatically if it does not exist. + pub network_name: String, + /// SSH listen address passed to the sandbox binary. + pub ssh_listen_addr: String, + /// SSH port inside the container. + pub ssh_port: u16, + /// Shared secret for the NSSH1 SSH handshake. + pub ssh_handshake_secret: String, + /// Maximum clock skew in seconds for SSH handshake timestamps. + pub ssh_handshake_skew_secs: u64, + /// Container stop timeout in seconds (SIGTERM → SIGKILL). + pub stop_timeout_secs: u32, + /// OCI image containing the openshell-sandbox supervisor binary. + /// Mounted read-only into sandbox containers at /opt/openshell/bin + /// using Podman's `type=image` mount. + pub supervisor_image: String, +} + +impl PodmanComputeConfig { + /// Resolve the default socket path from the environment. + /// + /// Uses `$XDG_RUNTIME_DIR` when available (set by `pam_systemd`/logind), + /// otherwise falls back to the real UID via `getuid()` — matching how the + /// Podman CLI itself resolves the rootless socket path. + #[must_use] + pub fn default_socket_path() -> PathBuf { + if let Ok(xdg) = std::env::var("XDG_RUNTIME_DIR") { + PathBuf::from(xdg).join("podman/podman.sock") + } else { + // Use the real UID from the kernel — reliable in containers, + // systemd services, CI, and after su/sudo. + let uid = nix::unistd::getuid(); + PathBuf::from(format!("/run/user/{uid}/podman/podman.sock")) + } + } +} + +impl Default for PodmanComputeConfig { + fn default() -> Self { + Self { + socket_path: Self::default_socket_path(), + default_image: String::new(), + image_pull_policy: ImagePullPolicy::default(), + grpc_endpoint: String::new(), + gateway_port: openshell_core::config::DEFAULT_SERVER_PORT, + sandbox_ssh_socket_path: "/run/openshell/ssh.sock".to_string(), + network_name: DEFAULT_NETWORK_NAME.to_string(), + ssh_listen_addr: String::new(), + ssh_port: DEFAULT_SSH_PORT, + ssh_handshake_secret: String::new(), + ssh_handshake_skew_secs: DEFAULT_SSH_HANDSHAKE_SKEW_SECS, + stop_timeout_secs: DEFAULT_STOP_TIMEOUT_SECS, + supervisor_image: DEFAULT_SUPERVISOR_IMAGE.to_string(), + } + } +} + +impl std::fmt::Debug for PodmanComputeConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PodmanComputeConfig") + .field("socket_path", &self.socket_path) + .field("default_image", &self.default_image) + .field("image_pull_policy", &self.image_pull_policy.as_str()) + .field("grpc_endpoint", &self.grpc_endpoint) + .field("gateway_port", &self.gateway_port) + .field("sandbox_ssh_socket_path", &self.sandbox_ssh_socket_path) + .field("network_name", &self.network_name) + .field("ssh_listen_addr", &self.ssh_listen_addr) + .field("ssh_port", &self.ssh_port) + .field("ssh_handshake_secret", &"[REDACTED]") + .field("ssh_handshake_skew_secs", &self.ssh_handshake_skew_secs) + .field("stop_timeout_secs", &self.stop_timeout_secs) + .field("supervisor_image", &self.supervisor_image) + .finish() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Serialises env-mutating tests so that parallel test threads cannot + /// observe each other's changes to `XDG_RUNTIME_DIR`. + static ENV_LOCK: std::sync::LazyLock> = + std::sync::LazyLock::new(|| std::sync::Mutex::new(())); + + #[test] + fn default_socket_path_respects_xdg_runtime_dir() { + let _guard = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + temp_env::with_vars([("XDG_RUNTIME_DIR", Some("/tmp/test-xdg"))], || { + let path = PodmanComputeConfig::default_socket_path(); + assert_eq!(path, PathBuf::from("/tmp/test-xdg/podman/podman.sock")); + }); + } + + #[test] + fn default_socket_path_falls_back_to_uid() { + let _guard = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + temp_env::with_vars([("XDG_RUNTIME_DIR", None::<&str>)], || { + let path = PodmanComputeConfig::default_socket_path(); + let uid = nix::unistd::getuid(); + assert_eq!( + path, + PathBuf::from(format!("/run/user/{uid}/podman/podman.sock")) + ); + }); + } +} diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs new file mode 100644 index 0000000000..d992881fe3 --- /dev/null +++ b/crates/openshell-driver-podman/src/container.rs @@ -0,0 +1,830 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Container spec construction for the Podman driver. + +use crate::config::PodmanComputeConfig; +use openshell_core::config::CDI_GPU_DEVICE_ALL; +use openshell_core::proto::compute::v1::DriverSandbox; +use serde::Serialize; +use serde_json::Value; +use std::collections::BTreeMap; + +/// Label key for the sandbox ID. +pub const LABEL_SANDBOX_ID: &str = "openshell.sandbox-id"; +/// Label key for the sandbox name. +pub const LABEL_SANDBOX_NAME: &str = "openshell.sandbox-name"; +/// Label applied to all managed containers. +pub const LABEL_MANAGED: &str = "openshell.managed"; +/// Label filter string for list/event queries. +pub const LABEL_MANAGED_FILTER: &str = "openshell.managed=true"; + +/// Container name prefix to avoid collisions with user containers. +const CONTAINER_PREFIX: &str = "openshell-sandbox-"; + +/// Volume name prefix. +const VOLUME_PREFIX: &str = "openshell-sandbox-"; + +/// Build a Podman container name from the sandbox name. +#[must_use] +pub fn container_name(sandbox_name: &str) -> String { + format!("{CONTAINER_PREFIX}{sandbox_name}") +} + +/// Build the workspace volume name from the sandbox ID. +#[must_use] +pub fn volume_name(sandbox_id: &str) -> String { + format!("{VOLUME_PREFIX}{sandbox_id}-workspace") +} + +/// Podman secret name prefix. +const SECRET_PREFIX: &str = "openshell-handshake-"; + +/// Build the Podman secret name for a sandbox's SSH handshake secret. +#[must_use] +pub fn secret_name(sandbox_id: &str) -> String { + format!("{SECRET_PREFIX}{sandbox_id}") +} + +/// Truncate a container ID to 12 characters (standard short form). +#[must_use] +pub(crate) fn short_id(id: &str) -> String { + id.chars().take(12).collect() +} + +// --------------------------------------------------------------------------- +// Typed container spec structs for the Podman libpod create API. +// --------------------------------------------------------------------------- + +#[derive(Serialize)] +struct ContainerSpec { + name: String, + image: String, + labels: BTreeMap, + env: BTreeMap, + volumes: Vec, + image_volumes: Vec, + hostname: String, + /// Overrides the image's ENTRYPOINT. In Podman's libpod API, `command` + /// only overrides CMD (appended as args to the entrypoint). We must set + /// `entrypoint` explicitly so the supervisor binary runs directly, + /// regardless of what ENTRYPOINT the sandbox image defines. + entrypoint: Vec, + command: Vec, + user: String, + cap_drop: Vec, + cap_add: Vec, + no_new_privileges: bool, + seccomp_profile_path: String, + image_pull_policy: String, + healthconfig: HealthConfig, + resource_limits: ResourceLimits, + /// Env-type secrets: map of `ENV_VAR_NAME → secret_name`. + /// Podman's libpod SpecGenerator uses `secret_env` (a flat map) for + /// environment-variable injection, distinct from `secrets` which only + /// handles file-mounted secrets under `/run/secrets/`. + secret_env: BTreeMap, + stop_timeout: u32, + /// Extra /etc/hosts entries. Used to inject `host.containers.internal` + /// via Podman's `host-gateway` magic so sandbox containers can reach + /// the gateway server running on the host in rootless mode. + hostadd: Vec, + netns: NetNS, + networks: BTreeMap, + #[serde(skip_serializing_if = "Option::is_none")] + devices: Option>, + /// Extra mounts for the libpod SpecGenerator (e.g. tmpfs entries). + mounts: Vec, + /// Port mappings from host to container. Using host_port=0 requests an + /// ephemeral port, readable back from the inspect response. + portmappings: Vec, +} + +/// A port mapping entry for the libpod SpecGenerator. +#[derive(Serialize)] +struct PortMapping { + host_port: u16, + container_port: u16, + protocol: String, +} + +/// A mount entry for the libpod container create API `mounts` field. +/// +/// Unlike `volumes` (named Podman volumes) or `image_volumes` (OCI image +/// mounts resolved at the libpod layer), these mounts are passed to the +/// libpod SpecGenerator and support arbitrary mount types (e.g. tmpfs). +/// Field names must be lowercase to match the libpod JSON schema. +#[derive(Serialize)] +struct Mount { + #[serde(rename = "type")] + mount_type: String, + source: String, + destination: String, + #[serde(skip_serializing_if = "Vec::is_empty")] + options: Vec, +} + +/// A Podman image volume for the libpod container create API. +/// +/// Image volumes mount an OCI image's filesystem into a container without +/// running it. Podman resolves these at the libpod layer before generating +/// the OCI runtime spec, unlike `mounts` which are passed directly to the +/// OCI runtime (crun/runc). +#[derive(Serialize)] +struct ImageVolume { + source: String, + destination: String, + rw: bool, +} + +#[derive(Serialize)] +struct NamedVolume { + name: String, + dest: String, + options: Vec, +} + +#[derive(Serialize)] +struct HealthConfig { + test: Vec, + #[serde(rename = "Interval")] + interval: u64, + #[serde(rename = "Timeout")] + timeout: u64, + #[serde(rename = "Retries")] + retries: u32, + #[serde(rename = "StartPeriod")] + start_period: u64, +} + +#[derive(Serialize)] +struct ResourceLimits { + cpu: CpuLimits, + memory: MemoryLimits, +} + +#[derive(Serialize)] +struct CpuLimits { + quota: u64, + period: u64, +} + +#[derive(Serialize)] +struct MemoryLimits { + limit: u64, +} + +#[derive(Serialize)] +struct NetNS { + nsmode: String, +} + +#[derive(Serialize)] +struct NetworkAttachment {} + +#[derive(Serialize)] +struct LinuxDevice { + path: String, +} + +/// Default limits: 2 CPU cores (200000µs quota / 100000µs period), 4 GiB memory. +const DEFAULT_CPU_QUOTA: u64 = 200_000; +const DEFAULT_CPU_PERIOD: u64 = 100_000; +const DEFAULT_MEMORY_LIMIT: u64 = 4_294_967_296; // 4 GiB + +/// Resolve the OCI image reference for a sandbox, using the template image +/// if provided, otherwise the driver's default image. +#[must_use] +pub fn resolve_image<'a>(sandbox: &'a DriverSandbox, config: &'a PodmanComputeConfig) -> &'a str { + let spec = sandbox.spec.as_ref(); + let template = spec.and_then(|s| s.template.as_ref()); + template + .map(|t| t.image.as_str()) + .filter(|s| !s.is_empty()) + .unwrap_or(&config.default_image) +} + +/// Merge environment variables from user spec/template with required driver vars. +/// +/// User-supplied vars are inserted first so that the required driver +/// vars always win -- preventing spec/template overrides of security- +/// critical values like `OPENSHELL_ENDPOINT` or `OPENSHELL_SANDBOX_ID`. +fn build_env( + sandbox: &DriverSandbox, + config: &PodmanComputeConfig, + image: &str, +) -> BTreeMap { + let spec = sandbox.spec.as_ref(); + let template = spec.and_then(|s| s.template.as_ref()); + + let mut env: BTreeMap = BTreeMap::new(); + + // 1. User-supplied environment (lowest priority). + if let Some(s) = spec { + if !s.log_level.is_empty() { + env.insert("OPENSHELL_LOG_LEVEL".into(), s.log_level.clone()); + } + for (k, v) in &s.environment { + env.insert(k.clone(), v.clone()); + } + } + if let Some(t) = template { + for (k, v) in &t.environment { + env.insert(k.clone(), v.clone()); + } + } + + // 2. Required driver vars (highest priority -- always overwrite). + env.insert("OPENSHELL_SANDBOX".into(), sandbox.name.clone()); + env.insert("OPENSHELL_SANDBOX_ID".into(), sandbox.id.clone()); + env.insert("OPENSHELL_ENDPOINT".into(), config.grpc_endpoint.clone()); + env.insert( + "OPENSHELL_SSH_SOCKET_PATH".into(), + config.sandbox_ssh_socket_path.clone(), + ); + env.insert( + "OPENSHELL_SSH_LISTEN_ADDR".into(), + config.ssh_listen_addr.clone(), + ); + // NOTE: The SSH handshake secret is injected via a Podman secret + // (see the "secrets" field below) rather than a plaintext env var. + // This prevents exposure through `podman inspect`. + env.insert( + "OPENSHELL_SSH_HANDSHAKE_SKEW_SECS".into(), + config.ssh_handshake_skew_secs.to_string(), + ); + env.insert("OPENSHELL_CONTAINER_IMAGE".into(), image.to_string()); + env.insert("OPENSHELL_SANDBOX_COMMAND".into(), "sleep infinity".into()); + + env +} + +/// Merge labels from the sandbox template with required managed labels. +/// +/// User-supplied labels are inserted first so that the managed labels +/// always win -- preventing template overrides of internal tracking labels. +fn build_labels(sandbox: &DriverSandbox) -> BTreeMap { + let template = sandbox.spec.as_ref().and_then(|s| s.template.as_ref()); + + let mut labels: BTreeMap = BTreeMap::new(); + if let Some(t) = template { + for (k, v) in &t.labels { + labels.insert(k.clone(), v.clone()); + } + } + // Managed labels (highest priority -- always overwrite). + labels.insert(LABEL_SANDBOX_ID.into(), sandbox.id.clone()); + labels.insert(LABEL_SANDBOX_NAME.into(), sandbox.name.clone()); + labels.insert(LABEL_MANAGED.into(), "true".into()); + + labels +} + +/// Parse resource limits from the sandbox template, falling back to defaults. +fn build_resource_limits(sandbox: &DriverSandbox) -> ResourceLimits { + let resources = sandbox + .spec + .as_ref() + .and_then(|s| s.template.as_ref()) + .and_then(|t| t.resources.as_ref()); + + let cpu_micros = resources + .filter(|r| !r.cpu_limit.is_empty()) + .and_then(|r| parse_cpu_to_microseconds(&r.cpu_limit)) + .unwrap_or(DEFAULT_CPU_QUOTA); + + let mem_bytes = resources + .filter(|r| !r.memory_limit.is_empty()) + .and_then(|r| parse_memory_to_bytes(&r.memory_limit)) + .unwrap_or(DEFAULT_MEMORY_LIMIT); + + ResourceLimits { + cpu: CpuLimits { + quota: cpu_micros, + period: DEFAULT_CPU_PERIOD, + }, + memory: MemoryLimits { limit: mem_bytes }, + } +} + +/// Build CDI GPU device list if GPU is requested. +fn build_devices(sandbox: &DriverSandbox) -> Option> { + if sandbox.spec.as_ref().is_some_and(|s| s.gpu) { + Some(vec![LinuxDevice { + path: CDI_GPU_DEVICE_ALL.into(), + }]) + } else { + None + } +} + +/// Build the Podman container creation JSON spec. +#[must_use] +pub fn build_container_spec(sandbox: &DriverSandbox, config: &PodmanComputeConfig) -> Value { + let image = resolve_image(sandbox, config); + let name = container_name(&sandbox.name); + let vol = volume_name(&sandbox.id); + + let env = build_env(sandbox, config, image); + let labels = build_labels(sandbox); + let resource_limits = build_resource_limits(sandbox); + let devices = build_devices(sandbox); + + // Network configuration -- always bridge mode. + let mut networks = BTreeMap::new(); + networks.insert(config.network_name.clone(), NetworkAttachment {}); + + let container_spec = ContainerSpec { + name, + image: image.to_string(), + labels, + env, + volumes: vec![NamedVolume { + name: vol, + dest: "/sandbox".into(), + options: vec!["rw".into()], + }], + // Side-load the supervisor binary from a standalone OCI image. + // Podman resolves image_volumes at the libpod layer, mounting the + // image's filesystem at the destination path without starting a + // container from it. The supervisor image is FROM scratch with just + // the binary at /openshell-sandbox, so it appears at + // /opt/openshell/bin/openshell-sandbox. + image_volumes: vec![ImageVolume { + source: config.supervisor_image.clone(), + destination: "/opt/openshell/bin".into(), + rw: false, + }], + hostname: format!("sandbox-{}", sandbox.name), + // Override the image's ENTRYPOINT so the supervisor binary runs + // directly. Sandbox images (e.g. the community base image) set + // ENTRYPOINT ["/bin/bash"], and Podman's `command` field only + // overrides CMD — which gets appended as args to the entrypoint. + // Without this, the container would run `/bin/bash /opt/openshell/bin/openshell-sandbox` + // and bash would fail trying to interpret the binary as a script. + entrypoint: vec!["/opt/openshell/bin/openshell-sandbox".into()], + command: vec![], + // Force the supervisor to run as root (UID 0). Sandbox images may + // set a non-root USER directive (e.g. `USER sandbox`), but the + // supervisor needs root to create network namespaces, set up the + // proxy, and configure Landlock/seccomp. This matches the K8s + // driver's runAsUser: 0. + user: "0:0".into(), + // The sandbox supervisor needs these capabilities during startup: + // SYS_ADMIN – seccomp filter installation, namespace creation, Landlock + // NET_ADMIN – network namespace veth setup, IP/route configuration + // SYS_PTRACE – reading /proc//exe and ancestor walk for policy + // SYSLOG – reading /dev/kmsg for bypass-detection diagnostics + // SETUID – drop_privileges(): setuid() to the sandbox user + // SETGID – drop_privileges(): setgid() + initgroups() to the sandbox group + // DAC_READ_SEARCH – reading /proc//fd/ across UIDs for process + // identity resolution. In rootless Podman the supervisor + // runs as UID 0 inside a user namespace while sandbox + // processes run as the sandbox user. The kernel's + // proc_fd_permission() calls generic_permission() which + // denies cross-UID access to the dr-x------ fd directory + // unless CAP_DAC_READ_SEARCH is present. Without it the + // proxy cannot determine which binary made each outbound + // connection and all traffic is denied. + // SETUID/SETGID are needed in rootless Podman: cap_drop:ALL removes them from + // the bounding set even though uid=0 owns the user namespace. Without them, + // setuid/setgid fail EPERM and the supervisor cannot drop to the sandbox user. + cap_drop: vec!["ALL".into()], + cap_add: vec![ + "SYS_ADMIN".into(), + "NET_ADMIN".into(), + "SYS_PTRACE".into(), + "SYSLOG".into(), + "SETUID".into(), + "SETGID".into(), + "DAC_READ_SEARCH".into(), + ], + // Disable the container-level seccomp profile. The sandbox supervisor + // installs its own policy-aware BPF seccomp filter at runtime via + // seccompiler (two-phase: clone3 blocker + main filter). The runtime + // filter is more restrictive than Podman's default — it blocks 20+ + // dangerous syscalls and conditionally restricts socket domains based + // on network policy. The filter self-seals by blocking further + // seccomp(SET_MODE_FILTER) calls after installation. + // + // A container-level profile would interfere by blocking the landlock + // and seccomp syscalls the supervisor needs during setup, before it + // locks itself down. + no_new_privileges: true, + seccomp_profile_path: "unconfined".into(), + image_pull_policy: config.image_pull_policy.as_str().to_string(), + healthconfig: HealthConfig { + test: vec![ + "CMD-SHELL".into(), + format!( + "test -e /var/run/openshell-ssh-ready || test -S {} || ss -tlnp | grep -q :{}", + config.sandbox_ssh_socket_path, config.ssh_port + ), + ], + interval: 3_000_000_000, + timeout: 2_000_000_000, + retries: 10, + start_period: 5_000_000_000, + }, + resource_limits, + // Inject the SSH handshake secret via Podman's secret_env map so it + // does not appear in `podman inspect` output. The libpod SpecGenerator + // uses `secret_env` (map of env_var → secret_name) for env-type secrets, + // distinct from `secrets` which only handles file mounts under /run/secrets/. + // The secret is created by the driver before the container + // (see `PodmanComputeDriver::create_sandbox`). + secret_env: BTreeMap::from([( + "OPENSHELL_SSH_HANDSHAKE_SECRET".into(), + secret_name(&sandbox.id), + )]), + stop_timeout: config.stop_timeout_secs, + // Inject host.containers.internal into /etc/hosts so sandbox + // containers can reach the gateway server on the host. The + // "host-gateway" magic value tells Podman to resolve to the + // host's actual IP (pasta uses 169.254.1.2 in rootless mode). + // This is the Podman equivalent of Docker's host.docker.internal. + hostadd: vec!["host.containers.internal:host-gateway".into()], + netns: NetNS { + nsmode: "bridge".to_string(), + }, + networks, + devices, + // Mount a tmpfs at /run/netns so the sandbox supervisor can create + // named network namespaces via `ip netns add`. The `ip` command requires + // /run/netns to exist and be bind-mountable; in rootless Podman this + // directory does not exist on the host, so the mkdir inside the container + // fails with EPERM. A private tmpfs gives the supervisor its own writable + // /run/netns without needing host filesystem access. + mounts: vec![Mount { + mount_type: "tmpfs".into(), + source: "tmpfs".into(), + destination: "/run/netns".into(), + options: vec!["rw".into(), "nosuid".into(), "nodev".into()], + }], + // Publish the SSH port with host_port=0 to get an ephemeral host port. + // In rootless Podman the bridge network (10.89.x.x) is not routable from + // the host, so we must use the published host port on 127.0.0.1 instead. + portmappings: vec![PortMapping { + host_port: 0, + container_port: config.ssh_port, + protocol: "tcp".into(), + }], + }; + + serde_json::to_value(container_spec).expect("ContainerSpec serialization cannot fail") +} + +/// Parse a Kubernetes-style CPU quantity to cgroup quota microseconds +/// (for a 100ms period). +/// +/// Examples: `"500m"` → 50000, `"2"` → 200000, `"0.5"` → 50000. +fn parse_cpu_to_microseconds(quantity: &str) -> Option { + let micros = if let Some(millis_str) = quantity.strip_suffix('m') { + let millis: u64 = millis_str.parse().ok()?; + // quota = millis * period / 1000 + millis.checked_mul(100)? + } else { + let cores: f64 = quantity.parse().ok()?; + if cores <= 0.0 || cores.is_nan() || cores.is_infinite() { + return None; + } + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + let val = (cores * 100_000.0) as u64; + val + }; + // A quota of 0 microseconds is invalid — treat as no limit. + if micros == 0 { None } else { Some(micros) } +} + +/// Parse a Kubernetes-style memory quantity to bytes. +/// +/// Supports: `Ki`, `Mi`, `Gi`, `Ti` (binary) and `k`, `M`, `G`, `T` +/// (decimal), as well as plain byte values. +fn parse_memory_to_bytes(quantity: &str) -> Option { + let suffixes: &[(&str, u64)] = &[ + ("Ti", 1024 * 1024 * 1024 * 1024), + ("Gi", 1024 * 1024 * 1024), + ("Mi", 1024 * 1024), + ("Ki", 1024), + ("T", 1_000_000_000_000), + ("G", 1_000_000_000), + ("M", 1_000_000), + ("k", 1_000), + ]; + + for (suffix, multiplier) in suffixes { + if let Some(num_str) = quantity.strip_suffix(suffix) { + let num: u64 = num_str.parse().ok()?; + return num.checked_mul(*multiplier); + } + } + + // Plain bytes. + quantity.parse().ok() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_cpu_millicore() { + assert_eq!(parse_cpu_to_microseconds("500m"), Some(50_000)); + assert_eq!(parse_cpu_to_microseconds("1000m"), Some(100_000)); + assert_eq!(parse_cpu_to_microseconds("250m"), Some(25_000)); + } + + #[test] + fn parse_cpu_whole_cores() { + assert_eq!(parse_cpu_to_microseconds("1"), Some(100_000)); + assert_eq!(parse_cpu_to_microseconds("2"), Some(200_000)); + assert_eq!(parse_cpu_to_microseconds("0.5"), Some(50_000)); + } + + #[test] + fn parse_memory_binary_suffixes() { + assert_eq!(parse_memory_to_bytes("256Mi"), Some(256 * 1024 * 1024)); + assert_eq!(parse_memory_to_bytes("4Gi"), Some(4 * 1024 * 1024 * 1024)); + assert_eq!(parse_memory_to_bytes("1Ki"), Some(1024)); + } + + #[test] + fn parse_memory_decimal_suffixes() { + assert_eq!(parse_memory_to_bytes("1G"), Some(1_000_000_000)); + assert_eq!(parse_memory_to_bytes("500M"), Some(500_000_000)); + } + + #[test] + fn parse_memory_plain_bytes() { + assert_eq!(parse_memory_to_bytes("1048576"), Some(1_048_576)); + } + + #[test] + fn container_name_is_prefixed() { + assert_eq!(container_name("my-sandbox"), "openshell-sandbox-my-sandbox"); + } + + #[test] + fn volume_name_uses_id() { + assert_eq!( + volume_name("abc-123"), + "openshell-sandbox-abc-123-workspace" + ); + } + + #[test] + fn secret_name_uses_id() { + assert_eq!(secret_name("abc-123"), "openshell-handshake-abc-123"); + } + + #[test] + fn short_id_truncates() { + assert_eq!(short_id("abc123def456789"), "abc123def456"); + assert_eq!(short_id("short"), "short"); + } + + #[test] + fn container_spec_includes_required_capabilities() { + let sandbox = test_sandbox("test-id", "test-name"); + let config = test_config(); + let spec = build_container_spec(&sandbox, &config); + + let cap_add = spec["cap_add"] + .as_array() + .expect("cap_add should be an array"); + let caps: Vec<&str> = cap_add.iter().filter_map(|v| v.as_str()).collect(); + assert!(caps.contains(&"SYS_ADMIN"), "missing SYS_ADMIN"); + assert!(caps.contains(&"NET_ADMIN"), "missing NET_ADMIN"); + assert!(caps.contains(&"SYS_PTRACE"), "missing SYS_PTRACE"); + assert!(caps.contains(&"SYSLOG"), "missing SYSLOG"); + assert!(caps.contains(&"SETUID"), "missing SETUID"); + assert!(caps.contains(&"SETGID"), "missing SETGID"); + assert!(caps.contains(&"DAC_READ_SEARCH"), "missing DAC_READ_SEARCH"); + } + + #[test] + fn container_spec_uses_secret_env_not_plaintext() { + let sandbox = test_sandbox("test-id", "test-name"); + let config = test_config(); + let spec = build_container_spec(&sandbox, &config); + + // The handshake secret must NOT appear in the plaintext env map. + let env_map = spec["env"].as_object().expect("env should be an object"); + assert!( + !env_map.contains_key("OPENSHELL_SSH_HANDSHAKE_SECRET"), + "handshake secret should not be in plaintext env" + ); + + // It should appear in secret_env (the libpod env-type secret map) instead. + let secret_env = spec["secret_env"] + .as_object() + .expect("secret_env should be an object"); + assert!( + secret_env.contains_key("OPENSHELL_SSH_HANDSHAKE_SECRET"), + "secret_env should map OPENSHELL_SSH_HANDSHAKE_SECRET to its secret name" + ); + assert_eq!( + secret_env["OPENSHELL_SSH_HANDSHAKE_SECRET"].as_str(), + Some("openshell-handshake-test-id"), + "secret_env value should be the Podman secret name for the sandbox" + ); + } + + #[test] + fn container_spec_sets_sandbox_name_in_env() { + let sandbox = test_sandbox("test-id", "my-sandbox"); + let config = test_config(); + let spec = build_container_spec(&sandbox, &config); + + let env_map = spec["env"].as_object().expect("env should be an object"); + assert_eq!( + env_map.get("OPENSHELL_SANDBOX").and_then(|v| v.as_str()), + Some("my-sandbox"), + ); + } + + #[test] + fn container_spec_sets_ssh_socket_path_in_env() { + let sandbox = test_sandbox("test-id", "test-name"); + let config = test_config(); + let spec = build_container_spec(&sandbox, &config); + + let env_map = spec["env"].as_object().expect("env should be an object"); + assert_eq!( + env_map + .get("OPENSHELL_SSH_SOCKET_PATH") + .and_then(|v| v.as_str()), + Some("/run/openshell/test-ssh.sock"), + ); + } + + #[test] + fn container_spec_healthcheck_accepts_supervisor_socket() { + let sandbox = test_sandbox("test-id", "test-name"); + let config = test_config(); + let spec = build_container_spec(&sandbox, &config); + + let healthcheck = spec["healthconfig"]["test"] + .as_array() + .expect("healthcheck test should be an array"); + let command = healthcheck + .get(1) + .and_then(|v| v.as_str()) + .expect("healthcheck should include shell command"); + assert!( + command.contains("test -S /run/openshell/test-ssh.sock"), + "healthcheck should consider the supervisor Unix socket ready" + ); + } + + #[test] + fn container_spec_required_vars_cannot_be_overridden() { + use openshell_core::proto::compute::v1::{DriverSandboxSpec, DriverSandboxTemplate}; + + let mut sandbox = test_sandbox("test-id", "legit-name"); + let mut env_overrides = std::collections::HashMap::new(); + env_overrides.insert( + "OPENSHELL_ENDPOINT".to_string(), + "http://evil.example.com".to_string(), + ); + env_overrides.insert("OPENSHELL_SANDBOX_ID".to_string(), "spoofed-id".to_string()); + env_overrides.insert( + "OPENSHELL_SSH_SOCKET_PATH".to_string(), + "/tmp/evil.sock".to_string(), + ); + sandbox.spec = Some(DriverSandboxSpec { + environment: env_overrides, + template: Some(DriverSandboxTemplate::default()), + ..Default::default() + }); + + let config = test_config(); + let spec = build_container_spec(&sandbox, &config); + + let env_map = spec["env"].as_object().expect("env should be an object"); + + assert_eq!( + env_map.get("OPENSHELL_ENDPOINT").and_then(|v| v.as_str()), + Some("http://localhost:50051"), + "OPENSHELL_ENDPOINT must not be overridden by user env" + ); + assert_eq!( + env_map.get("OPENSHELL_SANDBOX_ID").and_then(|v| v.as_str()), + Some("test-id"), + "OPENSHELL_SANDBOX_ID must not be overridden by user env" + ); + assert_eq!( + env_map + .get("OPENSHELL_SSH_SOCKET_PATH") + .and_then(|v| v.as_str()), + Some("/run/openshell/test-ssh.sock"), + "OPENSHELL_SSH_SOCKET_PATH must not be overridden by user env" + ); + } + + #[test] + fn container_spec_required_labels_cannot_be_overridden() { + use openshell_core::proto::compute::v1::{DriverSandboxSpec, DriverSandboxTemplate}; + + let mut sandbox = test_sandbox("real-id", "real-name"); + let mut label_overrides = std::collections::HashMap::new(); + label_overrides.insert("openshell.sandbox-id".to_string(), "spoofed-id".to_string()); + label_overrides.insert( + "openshell.sandbox-name".to_string(), + "spoofed-name".to_string(), + ); + sandbox.spec = Some(DriverSandboxSpec { + template: Some(DriverSandboxTemplate { + labels: label_overrides, + ..Default::default() + }), + ..Default::default() + }); + + let config = test_config(); + let spec = build_container_spec(&sandbox, &config); + + let labels = spec["labels"] + .as_object() + .expect("labels should be an object"); + assert_eq!( + labels.get("openshell.sandbox-id").and_then(|v| v.as_str()), + Some("real-id"), + "openshell.sandbox-id must not be overridden by template labels" + ); + assert_eq!( + labels + .get("openshell.sandbox-name") + .and_then(|v| v.as_str()), + Some("real-name"), + "openshell.sandbox-name must not be overridden by template labels" + ); + } + + #[test] + fn parse_cpu_negative_returns_none() { + assert_eq!(parse_cpu_to_microseconds("-1"), None); + assert_eq!(parse_cpu_to_microseconds("-500m"), None); + } + + #[test] + fn parse_cpu_zero_returns_none() { + assert_eq!(parse_cpu_to_microseconds("0m"), None); + assert_eq!(parse_cpu_to_microseconds("0"), None); + } + + fn test_sandbox(id: &str, name: &str) -> DriverSandbox { + DriverSandbox { + id: id.to_string(), + name: name.to_string(), + namespace: String::new(), + spec: None, + status: None, + } + } + + fn test_config() -> PodmanComputeConfig { + PodmanComputeConfig { + socket_path: std::path::PathBuf::from("/tmp/test.sock"), + default_image: "test-image:latest".to_string(), + grpc_endpoint: "http://localhost:50051".to_string(), + sandbox_ssh_socket_path: "/run/openshell/test-ssh.sock".to_string(), + ssh_listen_addr: "0.0.0.0:2222".to_string(), + ssh_handshake_secret: "test-secret-value".to_string(), + ..PodmanComputeConfig::default() + } + } + + #[test] + fn container_spec_includes_supervisor_image_volume() { + let sandbox = test_sandbox("test-id", "test-name"); + let config = test_config(); + let spec = build_container_spec(&sandbox, &config); + + let image_volumes = spec["image_volumes"] + .as_array() + .expect("image_volumes should be an array"); + assert_eq!( + image_volumes.len(), + 1, + "should have exactly one image volume" + ); + + let vol = &image_volumes[0]; + assert_eq!( + vol["source"].as_str(), + Some("openshell/supervisor:latest"), + "image volume source should be the supervisor image" + ); + assert_eq!( + vol["destination"].as_str(), + Some("/opt/openshell/bin"), + "image volume destination should be /opt/openshell/bin" + ); + assert_eq!( + vol["rw"].as_bool(), + Some(false), + "image volume should be read-only" + ); + } +} diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs new file mode 100644 index 0000000000..2f03159de0 --- /dev/null +++ b/crates/openshell-driver-podman/src/driver.rs @@ -0,0 +1,822 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Podman compute driver. + +use crate::client::{PodmanApiError, PodmanClient}; +use crate::config::PodmanComputeConfig; +use crate::container::{self, LABEL_MANAGED_FILTER, LABEL_SANDBOX_ID}; +use crate::watcher::{ + self, WatchStream, driver_sandbox_from_inspect, driver_sandbox_from_list_entry, +}; +use openshell_core::ComputeDriverError; +use openshell_core::proto::compute::v1::{DriverSandbox, GetCapabilitiesResponse}; +use tracing::{info, warn}; + +impl From for ComputeDriverError { + fn from(value: PodmanApiError) -> Self { + match value { + PodmanApiError::Conflict(_) => Self::AlreadyExists, + PodmanApiError::NotFound(msg) => Self::Message(format!("not found: {msg}")), + other => Self::Message(other.to_string()), + } + } +} + +/// Podman compute driver managing sandbox containers via the Podman REST API. +#[derive(Clone)] +pub struct PodmanComputeDriver { + client: PodmanClient, + config: PodmanComputeConfig, + /// The host's IP on the bridge network. Sandbox containers use this to + /// reach the gateway server when no explicit gRPC endpoint is configured. + network_gateway_ip: Option, +} + +impl std::fmt::Debug for PodmanComputeDriver { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PodmanComputeDriver") + .field("socket_path", &self.config.socket_path) + .field("default_image", &self.config.default_image) + .field("network_name", &self.config.network_name) + .finish() + } +} + +/// Construct and validate a container name from a sandbox name. +/// +/// Combines the prefix with the sandbox name and validates the result +/// against Podman's naming rules before any resources are created. +fn validated_container_name(sandbox_name: &str) -> Result { + let name = container::container_name(sandbox_name); + crate::client::validate_name(&name) + .map_err(|e| ComputeDriverError::Precondition(e.to_string()))?; + Ok(name) +} + +impl PodmanComputeDriver { + /// Create a new driver, verifying the Podman socket is reachable. + pub async fn new(mut config: PodmanComputeConfig) -> Result { + if !config.socket_path.exists() { + warn!( + path = %config.socket_path.display(), + "Podman socket not found; is the Podman service running? \ + Set OPENSHELL_PODMAN_SOCKET or XDG_RUNTIME_DIR to override." + ); + } + + let client = PodmanClient::new(config.socket_path.clone()); + + // Verify connectivity. + client.ping().await?; + + // Verify cgroups v2 and log system info. + match client.system_info().await { + Ok(info) => { + if info.host.cgroup_version != "v2" { + return Err(PodmanApiError::Connection(format!( + "cgroups v2 is required; detected cgroups '{}'. \ + Ensure your host uses a unified cgroup hierarchy \ + (systemd.unified_cgroup_hierarchy=1).", + info.host.cgroup_version + ))); + } + info!( + cgroup_version = %info.host.cgroup_version, + network_backend = %info.host.network_backend, + "Connected to Podman" + ); + } + Err(e) => { + return Err(PodmanApiError::Connection(format!( + "failed to query Podman system info: {e}" + ))); + } + } + + // Rootless pre-flight: warn if subuid/subgid ranges look missing. + // Not a hard error because some systems configure these via LDAP or + // other mechanisms that /etc/subuid does not reflect. + if nix::unistd::getuid().as_raw() != 0 { + check_subuid_range(); + } + + // Ensure the bridge network exists. + client.ensure_network(&config.network_name).await?; + let network_gateway_ip = client + .network_gateway_ip(&config.network_name) + .await + .unwrap_or(None); + info!( + network = %config.network_name, + gateway_ip = ?network_gateway_ip, + "Bridge network ready" + ); + + // Auto-detect the gRPC callback endpoint when not explicitly + // configured. Sandbox containers use host.containers.internal + // (injected via hostadd with host-gateway in the container spec) + // to reach the gateway server on the host. This works in both + // rootful and rootless Podman — the bridge gateway IP does NOT + // work in rootless mode because it lives inside the user + // namespace, not on the host. + if config.grpc_endpoint.is_empty() { + config.grpc_endpoint = + format!("http://host.containers.internal:{}", config.gateway_port); + info!( + grpc_endpoint = %config.grpc_endpoint, + "Auto-detected gRPC endpoint via host.containers.internal" + ); + } + + Ok(Self { + client, + config, + network_gateway_ip, + }) + } + + /// The host's IP on the bridge network, if available. + /// + /// Used by the server to auto-detect the gRPC callback endpoint when + /// no explicit `--grpc-endpoint` is configured. + #[must_use] + pub fn network_gateway_ip(&self) -> Option<&str> { + self.network_gateway_ip.as_deref() + } + + /// Report driver capabilities. + pub async fn capabilities(&self) -> Result { + let supports_gpu = self.has_gpu_capacity(); + Ok(GetCapabilitiesResponse { + driver_name: "podman".to_string(), + driver_version: openshell_core::VERSION.to_string(), + default_image: self.config.default_image.clone(), + supports_gpu, + }) + } + + #[must_use] + pub fn default_image(&self) -> &str { + &self.config.default_image + } + + /// Check whether GPU devices are available via CDI. + /// + /// The Podman system info response doesn't directly list CDI devices in all + /// versions. As a heuristic, check if the NVIDIA device node exists (this + /// works for both rootful and rootless). + fn has_gpu_capacity(&self) -> bool { + std::path::Path::new("/dev/nvidia0").exists() + } + + /// Validate a sandbox before creation. + pub async fn validate_sandbox_create( + &self, + sandbox: &DriverSandbox, + ) -> Result<(), ComputeDriverError> { + let gpu_requested = sandbox.spec.as_ref().is_some_and(|s| s.gpu); + if gpu_requested && !self.has_gpu_capacity() { + return Err(ComputeDriverError::Precondition( + "GPU sandbox requested, but no NVIDIA GPU devices are available.".to_string(), + )); + } + Ok(()) + } + + /// Create a sandbox container. + pub async fn create_sandbox(&self, sandbox: &DriverSandbox) -> Result<(), ComputeDriverError> { + if sandbox.name.is_empty() { + return Err(ComputeDriverError::Precondition( + "sandbox name is required".into(), + )); + } + if sandbox.id.is_empty() { + return Err(ComputeDriverError::Precondition( + "sandbox id is required".into(), + )); + } + + // Validate the composed container name early, before creating any + // resources (secret, volume), so we don't leave orphans when the + // name is invalid. + let name = validated_container_name(&sandbox.name)?; + + let vol_name = container::volume_name(&sandbox.id); + let sec_name = container::secret_name(&sandbox.id); + + info!( + sandbox_id = %sandbox.id, + sandbox_name = %sandbox.name, + container = %name, + "Creating sandbox container" + ); + + // 1a. Pull the supervisor image if needed. The supervisor binary + // is shipped in a standalone OCI image and mounted into sandbox + // containers via Podman's type=image mount. Using "missing" + // policy so the image is only pulled once and then cached. + info!( + image = %self.config.supervisor_image, + policy = "missing", + "Ensuring supervisor image" + ); + self.client + .pull_image(&self.config.supervisor_image, "missing") + .await + .map_err(ComputeDriverError::from)?; + + // 1b. Pull the sandbox image if needed (Podman does not pull on create). + let image = container::resolve_image(sandbox, &self.config); + if image.is_empty() { + return Err(ComputeDriverError::Precondition( + "no sandbox image configured: set --sandbox-image on the server \ + or provide an image in the sandbox template" + .to_string(), + )); + } + let pull_policy = self.config.image_pull_policy.as_str(); + info!(image = %image, policy = %pull_policy, "Ensuring sandbox image"); + self.client + .pull_image(image, pull_policy) + .await + .map_err(ComputeDriverError::from)?; + + // 2. Create the SSH handshake secret via the Podman secrets API + // so it is not exposed in `podman inspect` output. + self.client + .create_secret(&sec_name, self.config.ssh_handshake_secret.as_bytes()) + .await + .map_err(ComputeDriverError::from)?; + + // 3. Create workspace volume. + if let Err(e) = self.client.create_volume(&vol_name).await { + let _ = self.client.remove_secret(&sec_name).await; + return Err(ComputeDriverError::from(e)); + } + + // 4. Create container. + let spec = container::build_container_spec(sandbox, &self.config); + match self.client.create_container(&spec).await { + Ok(_) => {} + Err(PodmanApiError::Conflict(_)) => { + // Clean up the volume and secret we just created. They are + // keyed by *this* sandbox's ID, not the conflicting + // container's ID (which has the same name but a different + // ID), so they would be orphaned otherwise. + let _ = self.client.remove_volume(&vol_name).await; + let _ = self.client.remove_secret(&sec_name).await; + return Err(ComputeDriverError::AlreadyExists); + } + Err(e) => { + let _ = self.client.remove_volume(&vol_name).await; + let _ = self.client.remove_secret(&sec_name).await; + return Err(ComputeDriverError::from(e)); + } + } + + // 5. Start container. + if let Err(e) = self.client.start_container(&name).await { + warn!( + sandbox_name = %sandbox.name, + error = %e, + "Failed to start container; cleaning up" + ); + let _ = self.client.remove_container(&name).await; + let _ = self.client.remove_volume(&vol_name).await; + let _ = self.client.remove_secret(&sec_name).await; + return Err(ComputeDriverError::from(e)); + } + + info!( + sandbox_id = %sandbox.id, + sandbox_name = %sandbox.name, + "Sandbox container started" + ); + + Ok(()) + } + + /// Stop a sandbox container without deleting it. + pub async fn stop_sandbox(&self, sandbox_name: &str) -> Result<(), ComputeDriverError> { + let name = validated_container_name(sandbox_name)?; + info!(sandbox_name = %sandbox_name, container = %name, "Stopping sandbox container"); + + self.client + .stop_container(&name, self.config.stop_timeout_secs) + .await + .map_err(ComputeDriverError::from) + } + + /// Delete a sandbox container and its workspace volume. + pub async fn delete_sandbox( + &self, + sandbox_id: &str, + sandbox_name: &str, + ) -> Result { + if sandbox_id.is_empty() { + return Err(ComputeDriverError::Precondition( + "sandbox id is required".into(), + )); + } + let name = validated_container_name(sandbox_name)?; + info!( + sandbox_id = %sandbox_id, + sandbox_name = %sandbox_name, + container = %name, + "Deleting sandbox container" + ); + + // Use the request's stable sandbox ID as the source of truth for + // cleanup. Inspect is only used as a best-effort cross-check so + // cleanup still works if the container is already gone or mislabeled. + match self.client.inspect_container(&name).await { + Ok(inspect) => match inspect.config.labels.get(LABEL_SANDBOX_ID) { + Some(label_id) if label_id != sandbox_id => { + warn!( + sandbox_id = %sandbox_id, + sandbox_name = %sandbox_name, + container = %name, + label_sandbox_id = %label_id, + "Container label sandbox ID did not match delete request; cleaning up using request sandbox_id" + ); + } + None => { + warn!( + sandbox_id = %sandbox_id, + sandbox_name = %sandbox_name, + container = %name, + "Container missing '{}' label; cleaning up using request sandbox_id", + LABEL_SANDBOX_ID, + ); + } + Some(_) => {} + }, + Err(PodmanApiError::NotFound(_)) => {} + Err(e) => return Err(ComputeDriverError::from(e)), + } + + // Stop (best-effort). + let _ = self + .client + .stop_container(&name, self.config.stop_timeout_secs) + .await; + + // Remove container. If NotFound, the container was removed between + // inspect and here (TOCTOU race); proceed with volume/secret cleanup + // since those resources are idempotent to remove. + let container_existed = match self.client.remove_container(&name).await { + Ok(()) => true, + Err(PodmanApiError::NotFound(_)) => false, + Err(e) => return Err(ComputeDriverError::from(e)), + }; + + // Remove workspace volume and handshake secret. + let vol = container::volume_name(sandbox_id); + if let Err(e) = self.client.remove_volume(&vol).await { + warn!( + sandbox_id = %sandbox_id, + sandbox_name = %sandbox_name, + volume = %vol, + error = %e, + "Failed to remove workspace volume" + ); + } + let sec = container::secret_name(sandbox_id); + if let Err(e) = self.client.remove_secret(&sec).await { + warn!( + sandbox_id = %sandbox_id, + sandbox_name = %sandbox_name, + secret = %sec, + error = %e, + "Failed to remove handshake secret" + ); + } + + Ok(container_existed) + } + + /// Check whether a sandbox container exists. + pub async fn sandbox_exists(&self, sandbox_name: &str) -> Result { + let name = container::container_name(sandbox_name); + match self.client.inspect_container(&name).await { + Ok(_) => Ok(true), + Err(PodmanApiError::NotFound(_)) => Ok(false), + Err(e) => Err(ComputeDriverError::from(e)), + } + } + + /// Fetch a single sandbox by name. + pub async fn get_sandbox( + &self, + sandbox_name: &str, + ) -> Result, ComputeDriverError> { + let name = container::container_name(sandbox_name); + match self.client.inspect_container(&name).await { + Ok(inspect) => Ok(driver_sandbox_from_inspect(&inspect)), + Err(PodmanApiError::NotFound(_)) => Ok(None), + Err(e) => Err(ComputeDriverError::from(e)), + } + } + + /// List all managed sandboxes. + /// + /// Only inspects running containers (to get health status). Non-running + /// containers are built directly from the list entry data. + pub async fn list_sandboxes(&self) -> Result, ComputeDriverError> { + let entries = self + .client + .list_containers(LABEL_MANAGED_FILTER) + .await + .map_err(ComputeDriverError::from)?; + + let mut sandboxes = Vec::with_capacity(entries.len()); + for entry in &entries { + if entry.state == "running" { + // Running containers need inspect for health check status. + match self.client.inspect_container(&entry.id).await { + Ok(inspect) => { + if let Some(sandbox) = driver_sandbox_from_inspect(&inspect) { + sandboxes.push(sandbox); + continue; + } + } + Err(e) => { + let name = entry.names.first().cloned().unwrap_or_default(); + warn!( + container = %name, + error = %e, + "Failed to inspect running container during list, falling back to list entry" + ); + } + } + } + // Non-running containers (or inspect fallback): build from list data. + if let Some(sandbox) = driver_sandbox_from_list_entry(entry) { + sandboxes.push(sandbox); + } + } + + sandboxes.sort_by(|a, b| a.name.cmp(&b.name).then_with(|| a.id.cmp(&b.id))); + Ok(sandboxes) + } + + /// Start watching all managed sandbox containers. + pub async fn watch_sandboxes(&self) -> Result { + watcher::start_watch(self.client.clone()) + .await + .map_err(ComputeDriverError::from) + } +} + +#[cfg(test)] +impl PodmanComputeDriver { + pub(crate) fn for_tests(config: PodmanComputeConfig) -> Self { + let client = PodmanClient::new(config.socket_path.clone()); + Self { + client, + config, + network_gateway_ip: None, + } + } +} + +/// Check whether the current user has subuid/subgid ranges configured. +/// +/// Rootless Podman requires entries in `/etc/subuid` and `/etc/subgid` for +/// the running user. If missing, container creation fails with an obscure +/// error. This pre-flight check emits a warning to guide operators. +fn check_subuid_range() { + let uid = nix::unistd::getuid().as_raw(); + let username = nix::unistd::User::from_uid(nix::unistd::Uid::from_raw(uid)) + .ok() + .flatten() + .map(|u| u.name); + + let has_range = |path: &str| -> bool { + let Ok(content) = std::fs::read_to_string(path) else { + return false; + }; + let uid_str = uid.to_string(); + content.lines().any(|line| { + let Some(entry) = line.split(':').next() else { + return false; + }; + entry == uid_str || username.as_deref() == Some(entry) + }) + }; + + if !has_range("/etc/subuid") || !has_range("/etc/subgid") { + let user_display = username.as_deref().map_or_else( + || format!("UID {uid}"), + |name| format!("{name} (UID {uid})"), + ); + warn!( + user = %user_display, + "Rootless Podman detected but no /etc/subuid or /etc/subgid entry found. \ + Container creation may fail. Add entries with: \ + sudo usermod --add-subuids 100000-165535 --add-subgids 100000-165535 $(whoami)" + ); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use http_body_util::Full; + use hyper::body::Bytes; + use hyper::server::conn::http1; + use hyper::service::service_fn; + use hyper::{Response, StatusCode}; + use hyper_util::rt::TokioIo; + use std::collections::VecDeque; + use std::convert::Infallible; + use std::path::PathBuf; + use std::sync::{Arc, Mutex}; + use std::time::{SystemTime, UNIX_EPOCH}; + use tokio::net::UnixListener; + + #[test] + fn podman_driver_error_from_conflict() { + let err = ComputeDriverError::from(PodmanApiError::Conflict("exists".into())); + assert!(matches!(err, ComputeDriverError::AlreadyExists)); + } + + #[test] + fn podman_driver_error_from_not_found() { + let err = ComputeDriverError::from(PodmanApiError::NotFound("gone".into())); + assert!(matches!(err, ComputeDriverError::Message(_))); + } + + // ── gateway_port / grpc_endpoint auto-detection ─────────────────────── + // + // PodmanComputeDriver::new() fills grpc_endpoint when it is empty. + // These tests use for_tests() (which skips the Podman socket handshake) + // to verify the endpoint that ends up in the config — and therefore in + // OPENSHELL_ENDPOINT inside every sandbox container. + + #[test] + fn grpc_endpoint_auto_detected_from_gateway_port() { + let config = PodmanComputeConfig { + gateway_port: 8081, + ..PodmanComputeConfig::default() + }; + // Simulate what new() does once the socket/network checks pass. + let mut cfg = config; + if cfg.grpc_endpoint.is_empty() { + cfg.grpc_endpoint = format!("http://host.containers.internal:{}", cfg.gateway_port); + } + assert_eq!(cfg.grpc_endpoint, "http://host.containers.internal:8081"); + } + + #[test] + fn grpc_endpoint_auto_detected_uses_default_port_when_gateway_port_is_default() { + let config = PodmanComputeConfig::default(); + assert_eq!( + config.gateway_port, + openshell_core::config::DEFAULT_SERVER_PORT + ); + let mut cfg = config; + if cfg.grpc_endpoint.is_empty() { + cfg.grpc_endpoint = format!("http://host.containers.internal:{}", cfg.gateway_port); + } + assert_eq!(cfg.grpc_endpoint, "http://host.containers.internal:8080"); + } + + #[test] + fn explicit_grpc_endpoint_takes_precedence_over_gateway_port() { + let config = PodmanComputeConfig { + grpc_endpoint: "https://gateway.internal:9000".to_string(), + gateway_port: 8081, + ..PodmanComputeConfig::default() + }; + let mut cfg = config; + if cfg.grpc_endpoint.is_empty() { + cfg.grpc_endpoint = format!("http://host.containers.internal:{}", cfg.gateway_port); + } + assert_eq!(cfg.grpc_endpoint, "https://gateway.internal:9000"); + } + + #[derive(Clone)] + struct StubResponse { + status: StatusCode, + body: String, + } + + impl StubResponse { + fn new(status: StatusCode, body: impl Into) -> Self { + Self { + status, + body: body.into(), + } + } + } + + fn unique_socket_path(test_name: &str) -> PathBuf { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock should be after unix epoch") + .as_nanos(); + PathBuf::from(format!( + "/tmp/openshell-podman-{test_name}-{}-{nanos}.sock", + std::process::id() + )) + } + + fn test_driver(socket_path: PathBuf) -> PodmanComputeDriver { + let config = PodmanComputeConfig { + socket_path, + stop_timeout_secs: 10, + ..PodmanComputeConfig::default() + }; + PodmanComputeDriver::for_tests(config) + } + + fn api_path(path: &str) -> String { + format!("/v5.0.0{path}") + } + + fn spawn_podman_stub( + test_name: &str, + responses: Vec, + ) -> ( + PathBuf, + Arc>>, + tokio::task::JoinHandle<()>, + ) { + let socket_path = unique_socket_path(test_name); + let _ = std::fs::remove_file(&socket_path); + let listener = UnixListener::bind(&socket_path).expect("test socket should bind"); + let request_log = Arc::new(Mutex::new(Vec::new())); + let response_queue = Arc::new(Mutex::new(VecDeque::from(responses))); + let expected = response_queue + .lock() + .expect("response queue lock should not be poisoned") + .len(); + let socket_path_for_task = socket_path.clone(); + let log_for_task = request_log.clone(); + let queue_for_task = response_queue.clone(); + let handle = tokio::spawn(async move { + for _ in 0..expected { + let (stream, _) = listener.accept().await.expect("test stub should accept"); + let log = log_for_task.clone(); + let queue = queue_for_task.clone(); + http1::Builder::new() + .serve_connection( + TokioIo::new(stream), + service_fn(move |req| { + let log = log.clone(); + let queue = queue.clone(); + async move { + let path = req + .uri() + .path_and_query() + .map(|pq| pq.as_str().to_string()) + .unwrap_or_else(|| req.uri().path().to_string()); + log.lock() + .expect("request log lock should not be poisoned") + .push(format!("{} {}", req.method(), path)); + let response = queue + .lock() + .expect("response queue lock should not be poisoned") + .pop_front() + .expect("stub response should exist"); + Ok::<_, Infallible>( + Response::builder() + .status(response.status) + .body(Full::new(Bytes::from(response.body))) + .expect("stub response should build"), + ) + } + }), + ) + .await + .expect("test stub should serve request"); + } + let _ = std::fs::remove_file(&socket_path_for_task); + }); + (socket_path, request_log, handle) + } + + #[tokio::test] + async fn delete_sandbox_cleans_up_with_request_id_when_container_is_already_gone() { + let sandbox_id = "sandbox-123"; + let sandbox_name = "demo"; + let container_name = container::container_name(sandbox_name); + let volume_name = container::volume_name(sandbox_id); + let secret_name = container::secret_name(sandbox_id); + let (socket_path, request_log, handle) = spawn_podman_stub( + "delete-not-found", + vec![ + StubResponse::new(StatusCode::NOT_FOUND, r#"{"message":"gone"}"#), + StubResponse::new(StatusCode::NOT_FOUND, r#"{"message":"gone"}"#), + StubResponse::new(StatusCode::NOT_FOUND, r#"{"message":"gone"}"#), + StubResponse::new(StatusCode::NO_CONTENT, ""), + StubResponse::new(StatusCode::NO_CONTENT, ""), + ], + ); + let driver = test_driver(socket_path.clone()); + + let deleted = driver + .delete_sandbox(sandbox_id, sandbox_name) + .await + .expect("delete should succeed"); + + assert!(!deleted, "missing container should report deleted=false"); + handle.await.expect("stub task should finish"); + let requests = request_log + .lock() + .expect("request log lock should not be poisoned") + .clone(); + assert_eq!( + requests, + vec![ + format!( + "GET {}", + api_path(&format!("/libpod/containers/{container_name}/json")) + ), + format!( + "POST {}", + api_path(&format!( + "/libpod/containers/{container_name}/stop?timeout=10" + )) + ), + format!( + "DELETE {}", + api_path(&format!( + "/libpod/containers/{container_name}?force=true&v=true" + )) + ), + format!( + "DELETE {}", + api_path(&format!("/libpod/volumes/{volume_name}")) + ), + format!( + "DELETE {}", + api_path(&format!("/libpod/secrets/{secret_name}")) + ), + ] + ); + let _ = std::fs::remove_file(socket_path); + } + + #[tokio::test] + async fn delete_sandbox_uses_request_id_when_container_label_disagrees() { + let sandbox_id = "sandbox-request-id"; + let sandbox_name = "demo"; + let container_name = container::container_name(sandbox_name); + let volume_name = container::volume_name(sandbox_id); + let secret_name = container::secret_name(sandbox_id); + let inspect_body = serde_json::json!({ + "Id": "container-id", + "Name": format!("/{container_name}"), + "State": { + "Status": "running", + "Running": true + }, + "Config": { + "Labels": { + LABEL_SANDBOX_ID: "sandbox-label-id" + } + } + }) + .to_string(); + let (socket_path, request_log, handle) = spawn_podman_stub( + "delete-mismatch", + vec![ + StubResponse::new(StatusCode::OK, inspect_body), + StubResponse::new(StatusCode::NO_CONTENT, ""), + StubResponse::new(StatusCode::NO_CONTENT, ""), + StubResponse::new(StatusCode::NO_CONTENT, ""), + StubResponse::new(StatusCode::NO_CONTENT, ""), + ], + ); + let driver = test_driver(socket_path.clone()); + + let deleted = driver + .delete_sandbox(sandbox_id, sandbox_name) + .await + .expect("delete should succeed"); + + assert!(deleted, "existing container should report deleted=true"); + handle.await.expect("stub task should finish"); + let requests = request_log + .lock() + .expect("request log lock should not be poisoned") + .clone(); + assert_eq!( + requests[3..], + [ + format!( + "DELETE {}", + api_path(&format!("/libpod/volumes/{volume_name}")) + ), + format!( + "DELETE {}", + api_path(&format!("/libpod/secrets/{secret_name}")) + ), + ] + ); + let _ = std::fs::remove_file(socket_path); + } +} diff --git a/crates/openshell-driver-podman/src/grpc.rs b/crates/openshell-driver-podman/src/grpc.rs new file mode 100644 index 0000000000..95bf89d074 --- /dev/null +++ b/crates/openshell-driver-podman/src/grpc.rs @@ -0,0 +1,412 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use futures::{Stream, StreamExt}; +use openshell_core::proto::compute::v1::{ + CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, + GetCapabilitiesRequest, GetCapabilitiesResponse, GetSandboxRequest, GetSandboxResponse, + ListSandboxesRequest, ListSandboxesResponse, StopSandboxRequest, StopSandboxResponse, + ValidateSandboxCreateRequest, ValidateSandboxCreateResponse, WatchSandboxesEvent, + WatchSandboxesRequest, compute_driver_server::ComputeDriver, +}; +use std::pin::Pin; +use tonic::{Request, Response, Status}; + +use crate::PodmanComputeDriver; +use openshell_core::ComputeDriverError; + +#[derive(Debug, Clone)] +pub struct ComputeDriverService { + driver: PodmanComputeDriver, +} + +impl ComputeDriverService { + #[must_use] + pub fn new(driver: PodmanComputeDriver) -> Self { + Self { driver } + } +} + +#[tonic::async_trait] +impl ComputeDriver for ComputeDriverService { + async fn get_capabilities( + &self, + _request: Request, + ) -> Result, Status> { + self.driver + .capabilities() + .await + .map(Response::new) + .map_err(status_from_driver_error) + } + + async fn validate_sandbox_create( + &self, + request: Request, + ) -> Result, Status> { + let sandbox = request + .into_inner() + .sandbox + .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; + self.driver + .validate_sandbox_create(&sandbox) + .await + .map_err(status_from_driver_error)?; + Ok(Response::new(ValidateSandboxCreateResponse {})) + } + + async fn get_sandbox( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + if request.sandbox_name.is_empty() { + return Err(Status::invalid_argument("sandbox_name is required")); + } + + let sandbox = self + .driver + .get_sandbox(&request.sandbox_name) + .await + .map_err(status_from_driver_error)? + .ok_or_else(|| Status::not_found("sandbox not found"))?; + + if !request.sandbox_id.is_empty() && request.sandbox_id != sandbox.id { + return Err(Status::failed_precondition( + "sandbox_id did not match the fetched sandbox", + )); + } + + Ok(Response::new(GetSandboxResponse { + sandbox: Some(sandbox), + })) + } + + async fn list_sandboxes( + &self, + _request: Request, + ) -> Result, Status> { + let sandboxes = self + .driver + .list_sandboxes() + .await + .map_err(status_from_driver_error)?; + Ok(Response::new(ListSandboxesResponse { sandboxes })) + } + + async fn create_sandbox( + &self, + request: Request, + ) -> Result, Status> { + let sandbox = request + .into_inner() + .sandbox + .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; + self.driver + .create_sandbox(&sandbox) + .await + .map_err(status_from_driver_error)?; + Ok(Response::new(CreateSandboxResponse {})) + } + + async fn stop_sandbox( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + if request.sandbox_name.is_empty() { + return Err(Status::invalid_argument("sandbox_name is required")); + } + self.driver + .stop_sandbox(&request.sandbox_name) + .await + .map_err(status_from_driver_error)?; + Ok(Response::new(StopSandboxResponse {})) + } + + async fn delete_sandbox( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + if request.sandbox_id.is_empty() { + return Err(Status::invalid_argument("sandbox_id is required")); + } + if request.sandbox_name.is_empty() { + return Err(Status::invalid_argument("sandbox_name is required")); + } + let deleted = self + .driver + .delete_sandbox(&request.sandbox_id, &request.sandbox_name) + .await + .map_err(status_from_driver_error)?; + Ok(Response::new(DeleteSandboxResponse { deleted })) + } + + type WatchSandboxesStream = + Pin> + Send + 'static>>; + + async fn watch_sandboxes( + &self, + _request: Request, + ) -> Result, Status> { + let stream = self + .driver + .watch_sandboxes() + .await + .map_err(status_from_driver_error)?; + let stream = stream.map(|item| item.map_err(|err| Status::internal(err.to_string()))); + Ok(Response::new(Box::pin(stream))) + } +} + +fn status_from_driver_error(err: ComputeDriverError) -> Status { + match err { + ComputeDriverError::AlreadyExists => Status::already_exists("sandbox already exists"), + ComputeDriverError::Precondition(message) => Status::failed_precondition(message), + ComputeDriverError::Message(message) => Status::internal(message), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::PodmanComputeConfig; + use crate::container; + use http_body_util::Full; + use hyper::body::Bytes; + use hyper::server::conn::http1; + use hyper::service::service_fn; + use hyper::{Response as HyperResponse, StatusCode}; + use hyper_util::rt::TokioIo; + use std::collections::VecDeque; + use std::convert::Infallible; + use std::path::PathBuf; + use std::sync::{Arc, Mutex}; + use std::time::{SystemTime, UNIX_EPOCH}; + + #[test] + fn precondition_driver_errors_map_to_failed_precondition_status() { + let status = status_from_driver_error(ComputeDriverError::Precondition( + "sandbox container is not running".to_string(), + )); + + assert_eq!(status.code(), tonic::Code::FailedPrecondition); + assert_eq!(status.message(), "sandbox container is not running"); + } + + #[test] + fn already_exists_driver_errors_map_to_already_exists_status() { + let status = status_from_driver_error(ComputeDriverError::AlreadyExists); + assert_eq!(status.code(), tonic::Code::AlreadyExists); + } + + #[derive(Clone)] + struct StubResponse { + status: StatusCode, + body: String, + } + + impl StubResponse { + fn new(status: StatusCode, body: impl Into) -> Self { + Self { + status, + body: body.into(), + } + } + } + + fn unique_socket_path(test_name: &str) -> PathBuf { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock should be after unix epoch") + .as_nanos(); + PathBuf::from(format!( + "/tmp/openshell-podman-grpc-{test_name}-{}-{nanos}.sock", + std::process::id() + )) + } + + fn spawn_podman_stub( + test_name: &str, + responses: Vec, + ) -> ( + PathBuf, + Arc>>, + tokio::task::JoinHandle<()>, + ) { + let socket_path = unique_socket_path(test_name); + let _ = std::fs::remove_file(&socket_path); + let listener = + tokio::net::UnixListener::bind(&socket_path).expect("test socket should bind"); + let request_log = Arc::new(Mutex::new(Vec::new())); + let response_queue = Arc::new(Mutex::new(VecDeque::from(responses))); + let expected = response_queue + .lock() + .expect("response queue lock should not be poisoned") + .len(); + let socket_path_for_task = socket_path.clone(); + let log_for_task = request_log.clone(); + let queue_for_task = response_queue.clone(); + let handle = tokio::spawn(async move { + for _ in 0..expected { + let (stream, _) = listener.accept().await.expect("test stub should accept"); + let log = log_for_task.clone(); + let queue = queue_for_task.clone(); + http1::Builder::new() + .serve_connection( + TokioIo::new(stream), + service_fn(move |req| { + let log = log.clone(); + let queue = queue.clone(); + async move { + let path = req + .uri() + .path_and_query() + .map(|pq| pq.as_str().to_string()) + .unwrap_or_else(|| req.uri().path().to_string()); + log.lock() + .expect("request log lock should not be poisoned") + .push(format!("{} {}", req.method(), path)); + let response = queue + .lock() + .expect("response queue lock should not be poisoned") + .pop_front() + .expect("stub response should exist"); + Ok::<_, Infallible>( + HyperResponse::builder() + .status(response.status) + .body(Full::new(Bytes::from(response.body))) + .expect("stub response should build"), + ) + } + }), + ) + .await + .expect("test stub should serve request"); + } + let _ = std::fs::remove_file(&socket_path_for_task); + }); + (socket_path, request_log, handle) + } + + fn test_service(socket_path: PathBuf) -> ComputeDriverService { + let config = PodmanComputeConfig { + socket_path, + stop_timeout_secs: 10, + ..PodmanComputeConfig::default() + }; + ComputeDriverService::new(PodmanComputeDriver::for_tests(config)) + } + + fn api_path(path: &str) -> String { + format!("/v5.0.0{path}") + } + + #[tokio::test] + async fn delete_sandbox_rejects_missing_sandbox_name() { + let service = test_service(unique_socket_path("missing-name")); + + let err = ComputeDriver::delete_sandbox( + &service, + Request::new(DeleteSandboxRequest { + sandbox_id: "sandbox-123".to_string(), + sandbox_name: String::new(), + }), + ) + .await + .expect_err("missing sandbox_name should fail"); + + assert_eq!(err.code(), tonic::Code::InvalidArgument); + assert_eq!(err.message(), "sandbox_name is required"); + } + + #[tokio::test] + async fn delete_sandbox_rejects_missing_sandbox_id() { + let service = test_service(unique_socket_path("missing-id")); + + let err = ComputeDriver::delete_sandbox( + &service, + Request::new(DeleteSandboxRequest { + sandbox_id: String::new(), + sandbox_name: "demo".to_string(), + }), + ) + .await + .expect_err("missing sandbox_id should fail"); + + assert_eq!(err.code(), tonic::Code::InvalidArgument); + assert_eq!(err.message(), "sandbox_id is required"); + } + + #[tokio::test] + async fn delete_sandbox_forwards_request_sandbox_id_to_driver_cleanup() { + let sandbox_id = "sandbox-abc"; + let sandbox_name = "demo"; + let container_name = container::container_name(sandbox_name); + let volume_name = container::volume_name(sandbox_id); + let secret_name = container::secret_name(sandbox_id); + let (socket_path, request_log, handle) = spawn_podman_stub( + "forward-id", + vec![ + StubResponse::new(StatusCode::NOT_FOUND, r#"{"message":"gone"}"#), + StubResponse::new(StatusCode::NOT_FOUND, r#"{"message":"gone"}"#), + StubResponse::new(StatusCode::NOT_FOUND, r#"{"message":"gone"}"#), + StubResponse::new(StatusCode::NO_CONTENT, ""), + StubResponse::new(StatusCode::NO_CONTENT, ""), + ], + ); + let service = test_service(socket_path.clone()); + + let response = ComputeDriver::delete_sandbox( + &service, + Request::new(DeleteSandboxRequest { + sandbox_id: sandbox_id.to_string(), + sandbox_name: sandbox_name.to_string(), + }), + ) + .await + .expect("delete should succeed") + .into_inner(); + + assert!( + !response.deleted, + "already-removed containers should still report deleted=false" + ); + handle.await.expect("stub task should finish"); + let requests = request_log + .lock() + .expect("request log lock should not be poisoned") + .clone(); + assert_eq!( + requests, + vec![ + format!( + "GET {}", + api_path(&format!("/libpod/containers/{container_name}/json")) + ), + format!( + "POST {}", + api_path(&format!( + "/libpod/containers/{container_name}/stop?timeout=10" + )) + ), + format!( + "DELETE {}", + api_path(&format!( + "/libpod/containers/{container_name}?force=true&v=true" + )) + ), + format!( + "DELETE {}", + api_path(&format!("/libpod/volumes/{volume_name}")) + ), + format!( + "DELETE {}", + api_path(&format!("/libpod/secrets/{secret_name}")) + ), + ] + ); + let _ = std::fs::remove_file(socket_path); + } +} diff --git a/crates/openshell-driver-podman/src/lib.rs b/crates/openshell-driver-podman/src/lib.rs new file mode 100644 index 0000000000..630deaee1e --- /dev/null +++ b/crates/openshell-driver-podman/src/lib.rs @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +pub(crate) mod client; +pub mod config; +pub(crate) mod container; +pub mod driver; +pub mod grpc; +pub(crate) mod watcher; + +pub use config::PodmanComputeConfig; +pub use driver::PodmanComputeDriver; +pub use grpc::ComputeDriverService; diff --git a/crates/openshell-driver-podman/src/main.rs b/crates/openshell-driver-podman/src/main.rs new file mode 100644 index 0000000000..6020de5bde --- /dev/null +++ b/crates/openshell-driver-podman/src/main.rs @@ -0,0 +1,131 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use clap::Parser; +use miette::{IntoDiagnostic, Result}; +use std::net::SocketAddr; +use std::path::PathBuf; +use tracing::info; +use tracing_subscriber::EnvFilter; + +use openshell_core::VERSION; +use openshell_core::config::{ + DEFAULT_NETWORK_NAME, DEFAULT_SSH_HANDSHAKE_SKEW_SECS, DEFAULT_SSH_PORT, + DEFAULT_STOP_TIMEOUT_SECS, +}; +use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; +use openshell_driver_podman::config::ImagePullPolicy; +use openshell_driver_podman::{ComputeDriverService, PodmanComputeConfig, PodmanComputeDriver}; + +#[derive(Parser)] +#[command(name = "openshell-driver-podman")] +#[command(version = VERSION)] +struct Args { + #[arg( + long, + env = "OPENSHELL_COMPUTE_DRIVER_BIND", + default_value = "127.0.0.1:50061" + )] + bind_address: SocketAddr, + + #[arg(long, env = "OPENSHELL_LOG_LEVEL", default_value = "info")] + log_level: String, + + /// Path to the Podman API Unix socket. + #[arg(long, env = "OPENSHELL_PODMAN_SOCKET")] + podman_socket: Option, + + #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE")] + sandbox_image: Option, + + #[arg( + long, + env = "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY", + default_value_t = ImagePullPolicy::Missing + )] + sandbox_image_pull_policy: ImagePullPolicy, + + #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] + grpc_endpoint: Option, + + /// Port the gateway server is listening on. + /// + /// Used when `--grpc-endpoint` is not set to auto-detect the endpoint + /// that sandbox containers dial back to. + #[arg( + long, + env = "OPENSHELL_GATEWAY_PORT", + default_value_t = openshell_core::config::DEFAULT_SERVER_PORT + )] + gateway_port: u16, + + #[arg( + long, + env = "OPENSHELL_SANDBOX_SSH_SOCKET_PATH", + default_value = "/run/openshell/ssh.sock" + )] + sandbox_ssh_socket_path: String, + + /// Podman bridge network name. + #[arg(long, env = "OPENSHELL_NETWORK_NAME", default_value = DEFAULT_NETWORK_NAME)] + network_name: String, + + #[arg(long, env = "OPENSHELL_SANDBOX_SSH_PORT", default_value_t = DEFAULT_SSH_PORT)] + sandbox_ssh_port: u16, + + #[arg(long, env = "OPENSHELL_SSH_HANDSHAKE_SECRET")] + ssh_handshake_secret: String, + + #[arg(long, env = "OPENSHELL_SSH_HANDSHAKE_SKEW_SECS", default_value_t = DEFAULT_SSH_HANDSHAKE_SKEW_SECS)] + ssh_handshake_skew_secs: u64, + + /// Container stop timeout in seconds (SIGTERM → SIGKILL). + #[arg(long, env = "OPENSHELL_STOP_TIMEOUT", default_value_t = DEFAULT_STOP_TIMEOUT_SECS)] + stop_timeout: u32, + + /// OCI image containing the openshell-sandbox supervisor binary. + #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE")] + supervisor_image: String, +} + +#[tokio::main] +async fn main() -> Result<()> { + let args = Args::parse(); + tracing_subscriber::fmt() + .with_env_filter( + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)), + ) + .init(); + + let socket_path = args + .podman_socket + .unwrap_or_else(PodmanComputeConfig::default_socket_path); + + let driver = PodmanComputeDriver::new(PodmanComputeConfig { + socket_path, + default_image: args.sandbox_image.unwrap_or_default(), + image_pull_policy: args.sandbox_image_pull_policy, + grpc_endpoint: args.grpc_endpoint.unwrap_or_default(), + gateway_port: args.gateway_port, + sandbox_ssh_socket_path: args.sandbox_ssh_socket_path, + network_name: args.network_name, + ssh_listen_addr: format!("0.0.0.0:{}", args.sandbox_ssh_port), + ssh_port: args.sandbox_ssh_port, + ssh_handshake_secret: args.ssh_handshake_secret, + ssh_handshake_skew_secs: args.ssh_handshake_skew_secs, + stop_timeout_secs: args.stop_timeout, + supervisor_image: args.supervisor_image, + }) + .await + .into_diagnostic()?; + + info!(address = %args.bind_address, "Starting Podman compute driver"); + tonic::transport::Server::builder() + .add_service(ComputeDriverServer::new(ComputeDriverService::new(driver))) + .serve_with_shutdown(args.bind_address, async { + tokio::signal::ctrl_c().await.ok(); + info!("Received shutdown signal, draining in-flight requests"); + }) + .await + .into_diagnostic() +} diff --git a/crates/openshell-driver-podman/src/watcher.rs b/crates/openshell-driver-podman/src/watcher.rs new file mode 100644 index 0000000000..69cab6d884 --- /dev/null +++ b/crates/openshell-driver-podman/src/watcher.rs @@ -0,0 +1,550 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Maps Podman container events to the compute-driver watch protocol. + +use crate::client::{ + ContainerInspect, ContainerListEntry, ContainerState, HealthState, PodmanApiError, + PodmanClient, PodmanEvent, +}; +use crate::container::{LABEL_MANAGED_FILTER, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, short_id}; +use futures::Stream; +use openshell_core::ComputeDriverError; +use openshell_core::proto::compute::v1::{ + DriverCondition, DriverSandbox, DriverSandboxStatus, WatchSandboxesDeletedEvent, + WatchSandboxesEvent, WatchSandboxesSandboxEvent, watch_sandboxes_event, +}; +use std::pin::Pin; +use tokio::sync::mpsc; +use tokio_stream::wrappers::ReceiverStream; +use tracing::{debug, info, warn}; + +// Condition reason constants shared across event-building paths. +const CONDITION_RUNNING: &str = "ContainerRunning"; +const CONDITION_STARTING: &str = "ContainerStarting"; +const CONDITION_EXITED: &str = "ContainerExited"; +const CONDITION_STOPPED: &str = "ContainerStopped"; + +pub type WatchStream = + Pin> + Send>>; + +/// Build a `WatchSandboxesEvent` carrying a sandbox snapshot. +fn sandbox_event(sandbox: DriverSandbox) -> WatchSandboxesEvent { + WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Sandbox( + WatchSandboxesSandboxEvent { + sandbox: Some(sandbox), + }, + )), + } +} + +/// Build a `WatchSandboxesEvent` for a deleted sandbox. +fn deleted_event(sandbox_id: String) -> WatchSandboxesEvent { + WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Deleted( + WatchSandboxesDeletedEvent { sandbox_id }, + )), + } +} + +/// Start a watch stream that emits current state and live events. +/// +/// The stream first emits a snapshot of all currently-running managed +/// sandboxes (initial state sync), then delivers live container events +/// as they arrive from the Podman event stream. +/// +/// # Reconnection contract +/// +/// The returned stream is **single-use**. When the Podman event connection +/// drops (daemon restart, socket error, or clean shutdown), the stream +/// terminates with a final error item and stops producing events. +/// +/// Callers are responsible for reconnecting by calling [`start_watch`] again. +/// The server's `ComputeRuntime::watch_loop` in `openshell-server` provides +/// this behaviour with a 2-second backoff: when the stream terminates with an +/// error, `watch_loop` sleeps and then calls `watch_sandboxes()` again, which +/// ultimately calls `start_watch()` again and re-syncs state. +/// +/// **Do not add reconnection logic inside this function.** A local reconnect +/// would race with `watch_loop`'s retry and produce duplicate initial-sync +/// events that corrupt the server's sandbox index. +pub async fn start_watch(client: PodmanClient) -> Result { + let (tx, rx) = mpsc::channel::>(256); + + // 1. Subscribe to events first so we don't miss any during the list. + let mut event_rx = client.events_stream(LABEL_MANAGED_FILTER).await?; + + // 2. List existing containers for initial state sync. + let existing = client.list_containers(LABEL_MANAGED_FILTER).await?; + + for entry in &existing { + // For running containers, use inspect to get full state including + // health check status — matching the same condition derivation used + // for live events. + if entry.state == "running" { + match client.inspect_container(&entry.id).await { + Ok(inspect) => { + if let Some(sandbox) = driver_sandbox_from_inspect(&inspect) { + if tx.send(Ok(sandbox_event(sandbox))).await.is_err() { + return Err(PodmanApiError::Connection( + "watch receiver dropped during initial sync".into(), + )); + } + continue; + } + } + Err(e) => { + warn!( + container_id = %entry.id, + error = %e, + "Failed to inspect running container during initial sync, falling back to list entry" + ); + } + } + } + if let Some(event) = driver_sandbox_from_list_entry(entry).map(sandbox_event) { + if tx.send(Ok(event)).await.is_err() { + return Err(PodmanApiError::Connection( + "watch receiver dropped during initial sync".into(), + )); + } + } + } + + // 3. Stream live events (buffered during the list operation above). + + tokio::spawn(async move { + while let Some(result) = event_rx.recv().await { + match result { + Ok(event) => { + if let Some(we) = map_podman_event(&event, &client).await { + if tx.send(Ok(we)).await.is_err() { + return; + } + } + } + Err(e) => { + if tx + .send(Err(ComputeDriverError::Message(e.to_string()))) + .await + .is_err() + { + return; + } + } + } + } + // The Podman event stream has ended — either because the Podman + // daemon restarted, the Unix socket was closed, or the connection + // dropped. We do NOT reconnect here; see the doc-comment on + // start_watch() for the full reconnection contract. + // + // Sending this error terminates the WatchStream seen by the caller. + // The server's watch_loop detects the terminal error, waits 2 seconds, + // then calls watch_sandboxes() → start_watch() again, which re-lists + // all containers and re-subscribes to events. This ensures a full + // state resync after any Podman daemon interruption. + warn!("podman event stream ended unexpectedly; watch_loop will reconnect"); + let _ = tx + .send(Err(ComputeDriverError::Message( + "podman event stream ended unexpectedly".to_string(), + ))) + .await; + }); + + Ok(Box::pin(ReceiverStream::new(rx))) +} + +/// Map a Podman event to an optional watch event. +/// +/// Some events (like `remove`) produce a deletion event. State-change events +/// trigger a container inspect to build a full snapshot. +async fn map_podman_event( + event: &PodmanEvent, + client: &PodmanClient, +) -> Option { + let container_id = &event.actor.id; + let sandbox_id = event + .actor + .attributes + .get(LABEL_SANDBOX_ID) + .cloned() + .unwrap_or_default(); + + if sandbox_id.is_empty() { + debug!( + container_id = %container_id, + action = %event.action, + "Ignoring event for container without sandbox-id label" + ); + return None; + } + + match event.action.as_str() { + "remove" => Some(deleted_event(sandbox_id.clone())), + "create" | "start" | "stop" | "die" | "health_status" => { + // Inspect the container to get current state. + match client.inspect_container(container_id).await { + Ok(inspect) => driver_sandbox_from_inspect(&inspect).map(sandbox_event), + Err(PodmanApiError::NotFound(_)) => { + // The container is already gone by the time we inspected + // it. This is a normal race between the `die`/`stop` event + // and the subsequent `remove` event: Podman fires `die` + // first, but the container may be fully removed before we + // can inspect it. Treat this as a deletion so the server + // does not see a spurious phase regression. + info!( + container_id = %container_id, + action = %event.action, + "Container already removed when inspecting after event, emitting deleted event" + ); + Some(deleted_event(sandbox_id.clone())) + } + Err(e) => { + warn!( + container_id = %container_id, + error = %e, + "Failed to inspect container after event" + ); + // Emit a synthetic event with the info we have so the + // server knows something happened. + let sandbox_name = event + .actor + .attributes + .get(LABEL_SANDBOX_NAME) + .cloned() + .unwrap_or_default(); + Some(sandbox_event(build_driver_sandbox( + sandbox_id.clone(), + sandbox_name, + String::new(), + short_id(container_id), + DriverCondition { + r#type: "Ready".to_string(), + status: "Unknown".to_string(), + reason: "InspectFailed".to_string(), + message: format!("Container inspect failed: {e}"), + last_transition_time: String::new(), + }, + false, + ))) + } + } + } + _ => { + debug!(action = %event.action, "Ignoring unhandled Podman event"); + None + } + } +} + +/// Construct a `DriverSandbox` from common fields. +/// +/// Centralises the boilerplate that every event/inspect/list path shares: +/// `namespace`, `spec`, `agent_fd`, and `sandbox_fd` are always empty in +/// the Podman driver. +fn build_driver_sandbox( + sandbox_id: String, + sandbox_name: String, + instance_name: String, + instance_id: String, + condition: DriverCondition, + deleting: bool, +) -> DriverSandbox { + DriverSandbox { + id: sandbox_id, + name: sandbox_name, + namespace: String::new(), + spec: None, + status: Some(DriverSandboxStatus { + sandbox_name: instance_name, + instance_id, + agent_fd: String::new(), + sandbox_fd: String::new(), + conditions: vec![condition], + deleting, + }), + } +} + +/// Build a `DriverSandbox` from a container inspection result. +pub fn driver_sandbox_from_inspect(inspect: &ContainerInspect) -> Option { + let sandbox_id = inspect.config.labels.get(LABEL_SANDBOX_ID)?.clone(); + let sandbox_name = inspect + .config + .labels + .get(LABEL_SANDBOX_NAME) + .cloned() + .unwrap_or_default(); + + let condition = condition_from_state(&inspect.state); + let deleting = inspect.state.status == "removing"; + + Some(build_driver_sandbox( + sandbox_id, + sandbox_name, + inspect.name.trim_start_matches('/').to_string(), + short_id(&inspect.id), + condition, + deleting, + )) +} + +/// Build a `DriverSandbox` from a container list entry (no inspect needed). +pub(crate) fn driver_sandbox_from_list_entry(entry: &ContainerListEntry) -> Option { + let sandbox_id = entry.labels.get(LABEL_SANDBOX_ID)?.clone(); + let sandbox_name = entry + .labels + .get(LABEL_SANDBOX_NAME) + .cloned() + .unwrap_or_default(); + + let (reason, status_str, message) = match entry.state.as_str() { + "running" => ( + CONDITION_RUNNING, + "True", + "Container is running".to_string(), + ), + "created" => ("ContainerCreated", "False", String::new()), + "exited" => (CONDITION_EXITED, "False", String::new()), + "stopped" => (CONDITION_STOPPED, "False", String::new()), + "removing" => ("ContainerRemoving", "False", String::new()), + _ => ("Unknown", "Unknown", String::new()), + }; + + Some(build_driver_sandbox( + sandbox_id, + sandbox_name, + entry.names.first().cloned().unwrap_or_default(), + short_id(&entry.id), + DriverCondition { + r#type: "Ready".to_string(), + status: status_str.to_string(), + reason: reason.to_string(), + message, + last_transition_time: String::new(), + }, + entry.state == "removing", + )) +} + +/// Derive a `DriverCondition` from Podman container state. +fn condition_from_state(state: &ContainerState) -> DriverCondition { + let (status_val, reason, message) = match state.status.as_str() { + "running" => match &state.health { + Some(HealthState { status }) if status == "healthy" => { + ("True", "HealthCheckPassed", String::new()) + } + Some(HealthState { status }) if status == "unhealthy" => { + ("False", "HealthCheckFailed", String::new()) + } + Some(HealthState { status }) if status == "starting" => { + ("False", "HealthCheckStarting", String::new()) + } + _ => ("False", CONDITION_STARTING, String::new()), + }, + "created" => ("False", "ContainerCreated", String::new()), + "exited" | "stopped" => { + let msg = if state.oom_killed { + "Container was killed by the OOM killer".to_string() + } else { + format!("Container exited with code {}", state.exit_code) + }; + let reason = if state.oom_killed { + "OOMKilled" + } else { + CONDITION_EXITED + }; + ("False", reason, msg) + } + other => ( + "Unknown", + "Unknown", + format!("Unknown container state: {other}"), + ), + }; + + // Use Podman's state timestamps for last_transition_time: + // - Running/healthy states use started_at + // - Stopped/exited states use finished_at + let last_transition_time = match state.status.as_str() { + "running" => state.started_at.clone().unwrap_or_default(), + "exited" | "stopped" => state.finished_at.clone().unwrap_or_default(), + _ => String::new(), + }; + + DriverCondition { + r#type: "Ready".to_string(), + status: status_val.to_string(), + reason: reason.to_string(), + message, + last_transition_time, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn condition_healthy_container() { + let state = ContainerState { + status: "running".to_string(), + running: true, + exit_code: 0, + oom_killed: false, + health: Some(HealthState { + status: "healthy".to_string(), + }), + started_at: Some("2026-04-14T10:00:00Z".to_string()), + finished_at: None, + }; + let cond = condition_from_state(&state); + assert_eq!(cond.r#type, "Ready"); + assert_eq!(cond.status, "True"); + assert_eq!(cond.reason, "HealthCheckPassed"); + assert_eq!(cond.last_transition_time, "2026-04-14T10:00:00Z"); + } + + #[test] + fn condition_oom_killed() { + let state = ContainerState { + status: "exited".to_string(), + running: false, + exit_code: 137, + oom_killed: true, + health: None, + started_at: None, + finished_at: Some("2026-04-14T11:00:00Z".to_string()), + }; + let cond = condition_from_state(&state); + assert_eq!(cond.status, "False"); + assert_eq!(cond.reason, "OOMKilled"); + assert_eq!(cond.last_transition_time, "2026-04-14T11:00:00Z"); + } + + #[test] + fn condition_normal_exit() { + let state = ContainerState { + status: "exited".to_string(), + running: false, + exit_code: 1, + oom_killed: false, + health: None, + started_at: None, + finished_at: Some("2026-04-14T12:00:00Z".to_string()), + }; + let cond = condition_from_state(&state); + assert_eq!(cond.status, "False"); + assert_eq!(cond.reason, "ContainerExited"); + assert!(cond.message.contains("code 1")); + } + + #[test] + fn short_id_truncates() { + assert_eq!(short_id("abc123def456789"), "abc123def456"); + assert_eq!(short_id("short"), "short"); + } + + #[test] + fn sandbox_event_from_list_entry_running() { + let mut labels = std::collections::HashMap::new(); + labels.insert(LABEL_SANDBOX_ID.to_string(), "test-id".to_string()); + labels.insert(LABEL_SANDBOX_NAME.to_string(), "test-name".to_string()); + + let entry = ContainerListEntry { + id: "abc123def456789".to_string(), + names: vec!["openshell-sandbox-test-name".to_string()], + state: "running".to_string(), + labels, + ports: None, + networks: None, + exit_code: 0, + }; + + let sandbox = driver_sandbox_from_list_entry(&entry).expect("should produce a sandbox"); + let status = sandbox.status.expect("should have status"); + assert_eq!(status.conditions.len(), 1); + let cond = &status.conditions[0]; + assert_eq!(cond.status, "True"); + assert_eq!(cond.reason, "ContainerRunning"); + assert!(!status.deleting); + } + + #[test] + fn synthetic_inspect_failed_event_structure() { + // Verify the structure of an inspect-failure event by constructing one + // using the same pattern as the production code. + let condition = DriverCondition { + r#type: "Ready".to_string(), + status: "Unknown".to_string(), + reason: "InspectFailed".to_string(), + message: "Container inspect failed: connection refused".to_string(), + last_transition_time: String::new(), + }; + + let sandbox = DriverSandbox { + id: "sandbox-123".to_string(), + name: "test-sandbox".to_string(), + namespace: String::new(), + spec: None, + status: Some(DriverSandboxStatus { + sandbox_name: String::new(), + instance_id: short_id("container-id-full"), + agent_fd: String::new(), + sandbox_fd: String::new(), + conditions: vec![condition], + deleting: false, + }), + }; + + let event = WatchSandboxesEvent { + payload: Some(watch_sandboxes_event::Payload::Sandbox( + WatchSandboxesSandboxEvent { + sandbox: Some(sandbox), + }, + )), + }; + + let payload = event.payload.unwrap(); + let sandbox_event = match payload { + watch_sandboxes_event::Payload::Sandbox(s) => s, + _ => panic!("expected Sandbox payload"), + }; + let status = sandbox_event.sandbox.unwrap().status.unwrap(); + assert_eq!(status.conditions.len(), 1); + let cond = &status.conditions[0]; + assert_eq!(cond.reason, "InspectFailed"); + assert_eq!(cond.status, "Unknown"); + assert!(cond.message.contains("inspect failed")); + } + + #[test] + fn not_found_inspect_produces_deleted_event() { + // When inspect_container returns NotFound (404) after a `die` or + // `stop` event, the container is already gone. The watcher should emit + // a deleted_event rather than a synthetic InspectFailed sandbox event, + // preventing the server from regressing the sandbox phase back to + // Provisioning. + // + // We verify the shape of a deleted_event directly since + // map_podman_event is async and requires a live Podman client. The + // production code path is: + // Err(PodmanApiError::NotFound(_)) => Some(deleted_event(sandbox_id)) + let sandbox_id = "sandbox-abc-123".to_string(); + let event = deleted_event(sandbox_id.clone()); + + let payload = event.payload.expect("deleted event must have a payload"); + match payload { + watch_sandboxes_event::Payload::Deleted(d) => { + assert_eq!(d.sandbox_id, sandbox_id); + } + other => { + panic!( + "expected Deleted payload, got {other:?} — a NotFound inspect must not produce a sandbox or platform event" + ); + } + } + } +} diff --git a/crates/openshell-sandbox/src/sandbox/linux/netns.rs b/crates/openshell-sandbox/src/sandbox/linux/netns.rs index 37d11f0c3e..bbd02255f5 100644 --- a/crates/openshell-sandbox/src/sandbox/linux/netns.rs +++ b/crates/openshell-sandbox/src/sandbox/linux/netns.rs @@ -678,14 +678,26 @@ fn run_ip(args: &[&str]) -> Result<()> { Ok(()) } -/// Run an `ip netns exec` command inside a namespace. +/// Run an `ip` command inside a network namespace via `nsenter --net=`. +/// +/// We use `nsenter` instead of `ip netns exec` because `ip netns exec` +/// remounts `/sys` to reflect the target namespace's sysfs entries. That +/// sysfs remount requires real `CAP_SYS_ADMIN` in the host user namespace, +/// which is unavailable in rootless container runtimes (e.g. rootless +/// Podman). `nsenter --net=` enters only the network namespace without +/// changing the mount namespace, avoiding the sysfs remount entirely. +/// The supervisor's operations (addr add, link set, route add) are all +/// netlink-based and do not need sysfs access. fn run_ip_netns(netns: &str, args: &[&str]) -> Result<()> { - let mut full_args = vec!["netns", "exec", netns, "ip"]; + let ns_path = format!("/var/run/netns/{netns}"); + let net_flag = format!("--net={ns_path}"); + + let mut full_args = vec![net_flag.as_str(), "--", "ip"]; full_args.extend(args); - debug!(command = %format!("ip {}", full_args.join(" ")), "Running ip netns exec command"); + debug!(command = %format!("nsenter {}", full_args.join(" ")), "Running ip in namespace via nsenter"); - let output = Command::new("ip") + let output = Command::new("nsenter") .args(&full_args) .output() .into_diagnostic()?; @@ -693,8 +705,8 @@ fn run_ip_netns(netns: &str, args: &[&str]) -> Result<()> { if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); return Err(miette::miette!( - "ip netns exec {} ip {} failed: {}", - netns, + "nsenter --net={} ip {} failed: {}", + ns_path, args.join(" "), stderr.trim() )); @@ -703,17 +715,23 @@ fn run_ip_netns(netns: &str, args: &[&str]) -> Result<()> { Ok(()) } -/// Run an iptables command inside a network namespace. +/// Run an iptables command inside a network namespace via `nsenter --net=`. +/// +/// Uses `nsenter` instead of `ip netns exec` to avoid the sysfs remount +/// that fails in rootless container runtimes. See `run_ip_netns` for details. fn run_iptables_netns(netns: &str, iptables_cmd: &str, args: &[&str]) -> Result<()> { - let mut full_args = vec!["netns", "exec", netns, iptables_cmd]; + let ns_path = format!("/var/run/netns/{netns}"); + let net_flag = format!("--net={ns_path}"); + + let mut full_args = vec![net_flag.as_str(), "--", iptables_cmd]; full_args.extend(args); debug!( - command = %format!("ip {}", full_args.join(" ")), - "Running iptables in namespace" + command = %format!("nsenter {}", full_args.join(" ")), + "Running iptables in namespace via nsenter" ); - let output = Command::new("ip") + let output = Command::new("nsenter") .args(&full_args) .output() .into_diagnostic()?; @@ -721,8 +739,8 @@ fn run_iptables_netns(netns: &str, iptables_cmd: &str, args: &[&str]) -> Result< if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); return Err(miette::miette!( - "ip netns exec {} {} failed: {}", - netns, + "nsenter --net={} {} failed: {}", + ns_path, iptables_cmd, stderr.trim() )); diff --git a/crates/openshell-sandbox/src/ssh.rs b/crates/openshell-sandbox/src/ssh.rs index a8a402b3a6..4b2cd1572e 100644 --- a/crates/openshell-sandbox/src/ssh.rs +++ b/crates/openshell-sandbox/src/ssh.rs @@ -1445,4 +1445,73 @@ mod tests { .unwrap(); assert_eq!(rx_b.recv().unwrap(), b"still-alive"); } + + /// `install_pre_exec_no_pty` runs drop_privileges and succeeds when the + /// current user/group is already the configured one (no actual uid change). + /// + /// This exercises the pre_exec hook end-to-end without needing root: a policy + /// with no run_as_user/group is a no-op when the process is already unprivileged. + #[cfg(unix)] + #[test] + fn pre_exec_always_calls_drop_privileges() { + use crate::policy::{ + FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, SandboxPolicy, + }; + + // No user/group configured and not running as root → drop_privileges is + // a no-op, so spawn succeeds regardless of the effective UID. + let policy = SandboxPolicy { + version: 0, + filesystem: FilesystemPolicy::default(), + network: NetworkPolicy::default(), + landlock: LandlockPolicy::default(), + process: ProcessPolicy { + run_as_user: None, + run_as_group: None, + }, + }; + + // Skip if running as root: drop_privileges would try to switch to + // "sandbox" which may not exist in the test environment. + if nix::unistd::geteuid().is_root() { + return; + } + + let mut cmd = Command::new("echo"); + cmd.arg("drop-privileges-ok"); + cmd.stdout(Stdio::piped()); + + unsafe_pty::install_pre_exec_no_pty( + &mut cmd, + policy, + None, + None, // no netns fd + #[cfg(target_os = "linux")] + crate::sandbox::linux::prepare( + &SandboxPolicy { + version: 0, + filesystem: FilesystemPolicy::default(), + network: NetworkPolicy::default(), + landlock: LandlockPolicy::default(), + process: ProcessPolicy { + run_as_user: None, + run_as_group: None, + }, + }, + None, + ) + .expect("prepare should succeed in test environment"), + ); + + let output = cmd + .spawn() + .expect("spawn must succeed") + .wait_with_output() + .expect("wait_with_output"); + assert!(output.status.success(), "echo should exit 0"); + assert!( + String::from_utf8_lossy(&output.stdout).contains("drop-privileges-ok"), + "echo output should contain 'drop-privileges-ok'" + ); + } } diff --git a/crates/openshell-server/Cargo.toml b/crates/openshell-server/Cargo.toml index 6f8c949c78..c2c392f33e 100644 --- a/crates/openshell-server/Cargo.toml +++ b/crates/openshell-server/Cargo.toml @@ -17,6 +17,7 @@ path = "src/main.rs" [dependencies] openshell-core = { path = "../openshell-core" } openshell-driver-kubernetes = { path = "../openshell-driver-kubernetes" } +openshell-driver-podman = { path = "../openshell-driver-podman" } openshell-ocsf = { path = "../openshell-ocsf" } openshell-policy = { path = "../openshell-policy" } openshell-router = { path = "../openshell-router" } diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index 1ed5a59828..2e6e2823b1 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -6,6 +6,9 @@ use clap::{Command, CommandFactory, FromArgMatches, Parser}; use miette::{IntoDiagnostic, Result}; use openshell_core::ComputeDriverKind; +use openshell_core::config::{ + DEFAULT_SERVER_PORT, DEFAULT_SSH_HANDSHAKE_SKEW_SECS, DEFAULT_SSH_PORT, +}; use std::net::SocketAddr; use std::path::PathBuf; use tracing::info; @@ -20,7 +23,7 @@ use crate::{run_server, tracing_bus::TracingLogBus}; #[command(about = "OpenShell gRPC/HTTP server", long_about = None)] struct Args { /// Port to bind the server to (all interfaces). - #[arg(long, default_value_t = 8080, env = "OPENSHELL_SERVER_PORT")] + #[arg(long, default_value_t = DEFAULT_SERVER_PORT, env = "OPENSHELL_SERVER_PORT")] port: u16, /// Port for unauthenticated health endpoints (healthz, readyz). @@ -90,7 +93,7 @@ struct Args { ssh_gateway_host: String, /// Public port for the SSH gateway. - #[arg(long, env = "OPENSHELL_SSH_GATEWAY_PORT", default_value_t = 8080)] + #[arg(long, env = "OPENSHELL_SSH_GATEWAY_PORT", default_value_t = DEFAULT_SERVER_PORT)] ssh_gateway_port: u16, /// HTTP path for SSH CONNECT/upgrade. @@ -101,12 +104,15 @@ struct Args { )] ssh_connect_path: String, + /// SSH port inside sandbox pods. + #[arg(long, env = "OPENSHELL_SANDBOX_SSH_PORT", default_value_t = DEFAULT_SSH_PORT)] + sandbox_ssh_port: u16, /// Shared secret for gateway-to-sandbox SSH handshake. #[arg(long, env = "OPENSHELL_SSH_HANDSHAKE_SECRET")] ssh_handshake_secret: Option, /// Allowed clock skew in seconds for SSH handshake. - #[arg(long, env = "OPENSHELL_SSH_HANDSHAKE_SKEW_SECS", default_value_t = 300)] + #[arg(long, env = "OPENSHELL_SSH_HANDSHAKE_SKEW_SECS", default_value_t = DEFAULT_SSH_HANDSHAKE_SKEW_SECS)] ssh_handshake_skew_secs: u64, /// Kubernetes secret name containing client TLS materials for sandbox pods. @@ -271,6 +277,7 @@ async fn run_from_args(args: Args) -> Result<()> { .with_ssh_gateway_host(args.ssh_gateway_host) .with_ssh_gateway_port(args.ssh_gateway_port) .with_ssh_connect_path(args.ssh_connect_path) + .with_sandbox_ssh_port(args.sandbox_ssh_port) .with_ssh_handshake_skew_secs(args.ssh_handshake_skew_secs); if let Some(image) = args.sandbox_image { diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 35c72f80c1..19cfd5fafe 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -29,6 +29,9 @@ use openshell_core::proto::{ use openshell_driver_kubernetes::{ ComputeDriverService, KubernetesComputeConfig, KubernetesComputeDriver, }; +use openshell_driver_podman::{ + ComputeDriverService as PodmanDriverService, PodmanComputeConfig, PodmanComputeDriver, +}; use prost::Message; use std::fmt; use std::pin::Pin; @@ -50,15 +53,9 @@ const RECONCILE_INTERVAL: Duration = Duration::from_secs(60); /// corresponding backend resource before it is considered orphaned. const ORPHAN_GRACE_PERIOD: Duration = Duration::from_secs(300); -#[derive(Debug, thiserror::Error)] -pub enum ComputeError { - #[error("sandbox already exists")] - AlreadyExists, - #[error("{0}")] - Precondition(String), - #[error("{0}")] - Message(String), -} +// Re-export the shared error type under the name used by this module. +pub use openshell_core::ComputeDriverError as ComputeError; + #[derive(Debug)] pub(crate) struct ManagedDriverProcess { child: std::sync::Mutex>, @@ -208,6 +205,7 @@ impl ComputeRuntime { sandbox_watch_bus: SandboxWatchBus, tracing_log_bus: TracingLogBus, supervisor_sessions: Arc, + allows_loopback_endpoints: bool, ) -> Result { let default_image = driver .get_capabilities(Request::new(GetCapabilitiesRequest {})) @@ -248,6 +246,7 @@ impl ComputeRuntime { sandbox_watch_bus, tracing_log_bus, supervisor_sessions, + false, ) .await } @@ -270,6 +269,32 @@ impl ComputeRuntime { sandbox_watch_bus, tracing_log_bus, supervisor_sessions, + true, + ) + .await + } + + pub async fn new_podman( + config: PodmanComputeConfig, + store: Arc, + sandbox_index: SandboxIndex, + sandbox_watch_bus: SandboxWatchBus, + tracing_log_bus: TracingLogBus, + supervisor_sessions: Arc, + ) -> Result { + let driver = PodmanComputeDriver::new(config) + .await + .map_err(|err| ComputeError::Message(err.to_string()))?; + let driver: SharedComputeDriver = Arc::new(PodmanDriverService::new(driver)); + Self::from_driver( + driver, + None, + store, + sandbox_index, + sandbox_watch_bus, + tracing_log_bus, + supervisor_sessions, + true, ) .await } @@ -1147,7 +1172,14 @@ fn rewrite_user_facing_conditions(status: &mut Option, spec: Opti fn is_terminal_failure_reason(reason: &str) -> bool { let reason = reason.to_ascii_lowercase(); - let transient_reasons = ["reconcilererror", "dependenciesnotready", "starting"]; + let transient_reasons = [ + "reconcilererror", + "dependenciesnotready", + "starting", + "containerstarting", + "healthcheckstarting", + "inspectfailed", + ]; !transient_reasons.contains(&reason.as_str()) } diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 0442640d46..4a5ca55bd5 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -337,9 +337,53 @@ async fn build_compute_runtime( .await .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}"))) } - ComputeDriverKind::Podman => Err(Error::config( - "compute driver 'podman' is not implemented yet", - )), + ComputeDriverKind::Podman => { + let socket_path = std::env::var("OPENSHELL_PODMAN_SOCKET") + .ok() + .filter(|s| !s.is_empty()) + .map(std::path::PathBuf::from) + .unwrap_or_else(openshell_driver_podman::PodmanComputeConfig::default_socket_path); + + let network_name = std::env::var("OPENSHELL_NETWORK_NAME") + .ok() + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| openshell_core::config::DEFAULT_NETWORK_NAME.to_string()); + + let stop_timeout_secs: u32 = std::env::var("OPENSHELL_STOP_TIMEOUT") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(openshell_core::config::DEFAULT_STOP_TIMEOUT_SECS); + + let supervisor_image = std::env::var("OPENSHELL_SUPERVISOR_IMAGE") + .ok() + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| openshell_core::config::DEFAULT_SUPERVISOR_IMAGE.to_string()); + + ComputeRuntime::new_podman( + openshell_driver_podman::PodmanComputeConfig { + socket_path, + default_image: config.sandbox_image.clone(), + image_pull_policy: config.sandbox_image_pull_policy.parse().unwrap_or_default(), + grpc_endpoint: config.grpc_endpoint.clone(), + gateway_port: config.bind_address.port(), + sandbox_ssh_socket_path: config.sandbox_ssh_socket_path.clone(), + network_name, + ssh_listen_addr: format!("0.0.0.0:{}", config.sandbox_ssh_port), + ssh_port: config.sandbox_ssh_port, + ssh_handshake_secret: config.ssh_handshake_secret.clone(), + ssh_handshake_skew_secs: config.ssh_handshake_skew_secs, + stop_timeout_secs, + supervisor_image, + }, + store, + sandbox_index, + sandbox_watch_bus, + tracing_log_bus, + supervisor_sessions, + ) + .await + .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}"))) + } } } @@ -348,10 +392,11 @@ fn configured_compute_driver(config: &Config) -> Result { [] => Err(Error::config( "at least one compute driver must be configured", )), - [driver @ ComputeDriverKind::Kubernetes] | [driver @ ComputeDriverKind::Vm] => Ok(*driver), - [ComputeDriverKind::Podman] => Err(Error::config( - "compute driver 'podman' is not implemented yet", - )), + [ + driver @ (ComputeDriverKind::Kubernetes + | ComputeDriverKind::Vm + | ComputeDriverKind::Podman), + ] => Ok(*driver), drivers => Err(Error::config(format!( "multiple compute drivers are not supported yet; configured drivers: {}", drivers @@ -390,15 +435,7 @@ mod tests { } #[test] - fn configured_compute_driver_defaults_to_kubernetes() { - assert_eq!( - configured_compute_driver(&Config::new(None)).unwrap(), - ComputeDriverKind::Kubernetes - ); - } - - #[test] - fn configured_compute_driver_requires_at_least_one_entry() { + fn configured_compute_driver_rejects_empty_drivers() { let config = Config::new(None).with_compute_drivers([]); let err = configured_compute_driver(&config).unwrap_err(); assert!(err.to_string().contains("at least one compute driver")); @@ -417,12 +454,11 @@ mod tests { } #[test] - fn configured_compute_driver_rejects_unimplemented_driver() { + fn configured_compute_driver_accepts_podman() { let config = Config::new(None).with_compute_drivers([ComputeDriverKind::Podman]); - let err = configured_compute_driver(&config).unwrap_err(); - assert!( - err.to_string() - .contains("compute driver 'podman' is not implemented yet") + assert_eq!( + configured_compute_driver(&config).unwrap(), + ComputeDriverKind::Podman ); } diff --git a/crates/openshell-server/src/persistence/sqlite.rs b/crates/openshell-server/src/persistence/sqlite.rs index 3ee7799d79..3b660b3149 100644 --- a/crates/openshell-server/src/persistence/sqlite.rs +++ b/crates/openshell-server/src/persistence/sqlite.rs @@ -18,19 +18,27 @@ pub struct SqliteStore { impl SqliteStore { pub async fn connect(url: &str) -> Result { - let max_connections = if url.contains(":memory:") || url.contains("mode=memory") { - 1 - } else { - 5 - }; + let is_in_memory = url.contains(":memory:") || url.contains("mode=memory"); + let max_connections = if is_in_memory { 1 } else { 5 }; let options = SqliteConnectOptions::from_str(url) .map_err(|e| map_db_error(&e))? .create_if_missing(true); - let pool = SqlitePoolOptions::new() + let mut pool_options = SqlitePoolOptions::new() .max_connections(max_connections) - .min_connections(max_connections) + .min_connections(max_connections); + + // In-memory SQLite databases exist only while at least one connection + // is open. SQLx's default `max_lifetime` (30 min) and `idle_timeout` + // (10 min) would recycle the sole connection, destroying the database + // and all its tables. Disable both timeouts so the connection (and + // therefore the database) lives for the entire process lifetime. + if is_in_memory { + pool_options = pool_options.idle_timeout(None).max_lifetime(None); + } + + let pool = pool_options .connect_with(options) .await .map_err(|e| map_db_error(&e))?; diff --git a/crates/openshell-vm/scripts/build-rootfs.sh b/crates/openshell-vm/scripts/build-rootfs.sh index 23834c30e4..c55c004e03 100755 --- a/crates/openshell-vm/scripts/build-rootfs.sh +++ b/crates/openshell-vm/scripts/build-rootfs.sh @@ -33,6 +33,15 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# Source container engine abstraction (provides ce, ce_build, etc.) +_CE_SEARCH="${SCRIPT_DIR}/../../../tasks/scripts/container-engine.sh" +if [ -f "${_CE_SEARCH}" ]; then + source "${_CE_SEARCH}" +else + # Fallback: if run from a different working directory, try repo root. + source "$(cd "${SCRIPT_DIR}/../../.." && pwd)/tasks/scripts/container-engine.sh" +fi + # Source pinned dependency versions (digests, checksums, commit SHAs). # Environment variables override pins — see pins.env for details. PINS_FILE="${SCRIPT_DIR}/../pins.env" @@ -282,10 +291,10 @@ fi # ── Build base image with dependencies ───────────────────────────────── # Clean up any previous run -docker rm -f "${CONTAINER_NAME}" 2>/dev/null || true +ce rm -f "${CONTAINER_NAME}" 2>/dev/null || true echo "==> Building base image..." -docker build --platform "${DOCKER_PLATFORM}" -t "${BASE_IMAGE_TAG}" \ +ce build --platform "${DOCKER_PLATFORM}" -t "${BASE_IMAGE_TAG}" \ --build-arg "BASE_IMAGE=${VM_BASE_IMAGE}" -f - . <<'DOCKERFILE' ARG BASE_IMAGE FROM ${BASE_IMAGE} @@ -309,7 +318,7 @@ DOCKERFILE # Create a container and export the filesystem echo "==> Creating container..." -docker create --platform "${DOCKER_PLATFORM}" --name "${CONTAINER_NAME}" "${BASE_IMAGE_TAG}" /bin/true +ce create --platform "${DOCKER_PLATFORM}" --name "${CONTAINER_NAME}" "${BASE_IMAGE_TAG}" /bin/true echo "==> Exporting filesystem..." # Previous builds may leave overlayfs work/ dirs with permissions that @@ -319,9 +328,9 @@ if [ -d "${ROOTFS_DIR}" ]; then rm -rf "${ROOTFS_DIR}" fi mkdir -p "${ROOTFS_DIR}" -docker export "${CONTAINER_NAME}" | tar -C "${ROOTFS_DIR}" -xf - +ce export "${CONTAINER_NAME}" | tar -C "${ROOTFS_DIR}" -xf - -docker rm "${CONTAINER_NAME}" +ce rm "${CONTAINER_NAME}" # ── Inject k3s binary ──────────────────────────────────────────────── @@ -505,9 +514,9 @@ pull_and_save() { # Try to pull; if the registry is unavailable, fall back to the # local Docker image cache (image may exist from a previous pull). echo " pulling: ${image}..." - if ! docker pull --platform "${DOCKER_PLATFORM}" "${image}" --quiet 2>/dev/null; then - echo " pull failed, checking local Docker cache..." - if ! docker image inspect "${image}" >/dev/null 2>&1; then + if ! ce pull --platform "${DOCKER_PLATFORM}" "${image}" --quiet 2>/dev/null; then + echo " pull failed, checking local image cache..." + if ! ce image inspect "${image}" >/dev/null 2>&1; then echo "ERROR: image ${image} not available locally or from registry" exit 1 fi @@ -518,7 +527,7 @@ pull_and_save() { # Pipe through zstd for faster decompression and smaller tarballs. # k3s auto-imports .tar.zst files from the airgap images directory. # -T0 uses all CPU cores; -3 is a good speed/ratio tradeoff. - docker save "${image}" | zstd -T0 -3 -o "${output}" + ce save "${image}" | zstd -T0 -3 -o "${output}" # Cache for next rebuild. cp "${output}" "${cache}" } diff --git a/deploy/docker/Dockerfile.images b/deploy/docker/Dockerfile.images index a85a6a9ae3..f1fe490efe 100644 --- a/deploy/docker/Dockerfile.images +++ b/deploy/docker/Dockerfile.images @@ -49,6 +49,7 @@ COPY crates/openshell-bootstrap/Cargo.toml crates/openshell-bootstrap/Cargo.toml COPY crates/openshell-cli/Cargo.toml crates/openshell-cli/Cargo.toml COPY crates/openshell-core/Cargo.toml crates/openshell-core/Cargo.toml COPY crates/openshell-driver-kubernetes/Cargo.toml crates/openshell-driver-kubernetes/Cargo.toml +COPY crates/openshell-driver-podman/Cargo.toml crates/openshell-driver-podman/Cargo.toml COPY crates/openshell-ocsf/Cargo.toml crates/openshell-ocsf/Cargo.toml COPY crates/openshell-policy/Cargo.toml crates/openshell-policy/Cargo.toml COPY crates/openshell-providers/Cargo.toml crates/openshell-providers/Cargo.toml @@ -66,6 +67,7 @@ RUN mkdir -p \ crates/openshell-cli/src \ crates/openshell-core/src \ crates/openshell-driver-kubernetes/src \ + crates/openshell-driver-podman/src \ crates/openshell-ocsf/src \ crates/openshell-policy/src \ crates/openshell-providers/src \ @@ -80,6 +82,8 @@ RUN mkdir -p \ touch crates/openshell-core/src/lib.rs && \ touch crates/openshell-driver-kubernetes/src/lib.rs && \ printf 'fn main() {}\n' > crates/openshell-driver-kubernetes/src/main.rs && \ + touch crates/openshell-driver-podman/src/lib.rs && \ + printf 'fn main() {}\n' > crates/openshell-driver-podman/src/main.rs && \ touch crates/openshell-ocsf/src/lib.rs && \ touch crates/openshell-policy/src/lib.rs && \ touch crates/openshell-providers/src/lib.rs && \ @@ -115,6 +119,7 @@ ARG OPENSHELL_CARGO_VERSION COPY crates/openshell-core/ crates/openshell-core/ COPY crates/openshell-driver-kubernetes/ crates/openshell-driver-kubernetes/ +COPY crates/openshell-driver-podman/ crates/openshell-driver-podman/ COPY crates/openshell-ocsf/ crates/openshell-ocsf/ COPY crates/openshell-policy/ crates/openshell-policy/ COPY crates/openshell-providers/ crates/openshell-providers/ diff --git a/e2e/rust/e2e-podman.sh b/e2e/rust/e2e-podman.sh new file mode 100755 index 0000000000..38c6e6b7ce --- /dev/null +++ b/e2e/rust/e2e-podman.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Run the Rust e2e smoke test against a Podman-backed gateway. +# +# Usage: +# mise run e2e:podman # start a gateway with Podman driver +# mise run e2e:podman -- --port=9090 # use a specific port +# +# Options: +# --port=PORT Gateway listen port (default: random free port). +# +# The script: +# 1. Verifies Podman is available and the socket is reachable +# 2. Starts openshell-gateway with --drivers podman --disable-tls +# 3. Waits for the gateway to become healthy +# 4. Runs the Rust smoke test +# 5. Cleans up the gateway process and any leftover sandbox containers +# +# Prerequisites: +# - Rootless Podman service running (systemctl --user start podman.socket) +# - Supervisor sideload image built (mise run build:docker:supervisor-sideload) +# - Sandbox base image available locally + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +GATEWAY_BIN="${ROOT}/target/debug/openshell-gateway" +TIMEOUT=120 + +# ── Parse arguments ────────────────────────────────────────────────── +PORT="" +for arg in "$@"; do + case "$arg" in + --port=*) PORT="${arg#--port=}" ;; + *) echo "Unknown argument: $arg"; exit 1 ;; + esac +done + +if [ -z "${PORT}" ]; then + PORT=$(python3 -c 'import socket; s=socket.socket(); s.bind(("",0)); print(s.getsockname()[1]); s.close()') +fi + +# ── Pre-flight checks ─────────────────────────────────────────────── + +if ! command -v podman &>/dev/null; then + echo "ERROR: podman is not installed or not in PATH" + exit 1 +fi + +if ! podman info &>/dev/null; then + echo "ERROR: podman service is not reachable. Start it with:" + echo " systemctl --user start podman.socket" + exit 1 +fi + +if [ ! -f "${GATEWAY_BIN}" ]; then + echo "Building openshell-gateway..." + cargo build -p openshell-server --features openshell-core/dev-settings +fi + +# ── Resolve images ─────────────────────────────────────────────────── +# Use the same image defaults as the driver, allowing env overrides. +SUPERVISOR_IMAGE="${OPENSHELL_SUPERVISOR_IMAGE:-openshell/supervisor:dev}" +SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-}" + +# Verify the supervisor image exists locally. +if ! podman image exists "${SUPERVISOR_IMAGE}" 2>/dev/null; then + echo "ERROR: supervisor image '${SUPERVISOR_IMAGE}' not found locally." + echo "Build it with: mise run build:docker:supervisor-sideload" + exit 1 +fi + +# ── Generate a unique handshake secret ─────────────────────────────── +HANDSHAKE_SECRET="e2e-podman-$(head -c 16 /dev/urandom | xxd -p)" + +# ── Start the gateway ──────────────────────────────────────────────── +GW_LOG=$(mktemp /tmp/openshell-gw-podman-e2e.XXXXXX) +GW_PID="" + +cleanup() { + local exit_code=$? + + # Kill the gateway process. + if [ -n "${GW_PID:-}" ] && kill -0 "${GW_PID}" 2>/dev/null; then + echo "Stopping gateway (pid ${GW_PID})..." + kill "${GW_PID}" 2>/dev/null || true + wait "${GW_PID}" 2>/dev/null || true + fi + + # Clean up any leftover sandbox containers, volumes, and secrets. + echo "Cleaning up Podman resources..." + for cid in $(podman ps -a --filter label=openshell.managed=true --format '{{.ID}}' 2>/dev/null); do + podman rm -f "${cid}" 2>/dev/null || true + done + for vid in $(podman volume ls --filter label=openshell.managed=true --format '{{.Name}}' 2>/dev/null); do + podman volume rm -f "${vid}" 2>/dev/null || true + done + # Secrets created by the driver use the openshell-handshake- prefix. + for sid in $(podman secret ls --format '{{.Name}}' 2>/dev/null | grep '^openshell-handshake-'); do + podman secret rm "${sid}" 2>/dev/null || true + done + + if [ "${exit_code}" -ne 0 ] && [ -f "${GW_LOG}" ]; then + echo "=== Gateway log (preserved for debugging) ===" + cat "${GW_LOG}" + echo "=== end gateway log ===" + fi + + rm -f "${GW_LOG}" 2>/dev/null || true +} +trap cleanup EXIT + +echo "Starting openshell-gateway on port ${PORT} with Podman driver..." + +OPENSHELL_SSH_HANDSHAKE_SECRET="${HANDSHAKE_SECRET}" \ +OPENSHELL_SUPERVISOR_IMAGE="${SUPERVISOR_IMAGE}" \ + "${GATEWAY_BIN}" \ + --port "${PORT}" \ + --drivers podman \ + --disable-tls \ + --db-url "sqlite::memory:" \ + ${SANDBOX_IMAGE:+--sandbox-image "${SANDBOX_IMAGE}"} \ + --log-level info \ + >"${GW_LOG}" 2>&1 & +GW_PID=$! + +# ── Wait for health ───────────────────────────────────────────────── +echo "Waiting for gateway to become healthy (timeout ${TIMEOUT}s)..." +elapsed=0 +healthy=false +while [ "${elapsed}" -lt "${TIMEOUT}" ]; do + if ! kill -0 "${GW_PID}" 2>/dev/null; then + echo "ERROR: gateway exited before becoming ready" + cat "${GW_LOG}" + exit 1 + fi + + # Use curl to check the gateway's gRPC health endpoint. + # The gateway serves both gRPC and HTTP on the same port. + if curl -sf "http://127.0.0.1:${PORT}/healthz" >/dev/null 2>&1; then + healthy=true + break + fi + + sleep 2 + elapsed=$((elapsed + 2)) +done + +if [ "${healthy}" != "true" ]; then + echo "ERROR: gateway did not become healthy after ${TIMEOUT}s" + cat "${GW_LOG}" + exit 1 +fi +echo "Gateway is ready (${elapsed}s)." + +# ── Run the smoke test ─────────────────────────────────────────────── +export OPENSHELL_GATEWAY_ENDPOINT="http://127.0.0.1:${PORT}" +# Use a synthetic gateway name so the CLI does not require stored mTLS creds. +export OPENSHELL_GATEWAY="e2e-podman" +export OPENSHELL_PROVISION_TIMEOUT=300 + +echo "Running e2e smoke test (gateway: ${OPENSHELL_GATEWAY}, endpoint: ${OPENSHELL_GATEWAY_ENDPOINT})..." +cargo build -p openshell-cli --features openshell-core/dev-settings +cargo test --manifest-path e2e/rust/Cargo.toml --features e2e --test smoke -- --nocapture + +echo "Smoke test passed." diff --git a/e2e/rust/src/harness/sandbox.rs b/e2e/rust/src/harness/sandbox.rs index 3a9601a839..9141f8e2f3 100644 --- a/e2e/rust/src/harness/sandbox.rs +++ b/e2e/rust/src/harness/sandbox.rs @@ -74,9 +74,11 @@ impl SandboxGuard { } cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); - let output = cmd - .output() + let output = timeout(SANDBOX_READY_TIMEOUT, cmd.output()) .await + .map_err(|_| { + format!("sandbox create timed out after {SANDBOX_READY_TIMEOUT:?}") + })? .map_err(|e| format!("failed to spawn openshell: {e}"))?; let stdout = String::from_utf8_lossy(&output.stdout).to_string(); @@ -226,9 +228,13 @@ impl SandboxGuard { .args(command); cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); - let output = cmd - .output() + let output = timeout(SANDBOX_READY_TIMEOUT, cmd.output()) .await + .map_err(|_| { + format!( + "sandbox create --upload timed out after {SANDBOX_READY_TIMEOUT:?}" + ) + })? .map_err(|e| format!("failed to spawn openshell: {e}"))?; let stdout = String::from_utf8_lossy(&output.stdout).to_string(); diff --git a/e2e/rust/tests/no_proxy.rs b/e2e/rust/tests/no_proxy.rs index c05c513274..ced4d02d5f 100644 --- a/e2e/rust/tests/no_proxy.rs +++ b/e2e/rust/tests/no_proxy.rs @@ -48,7 +48,7 @@ finally: #[tokio::test] async fn sandbox_bypasses_proxy_for_localhost_http() { - let guard = SandboxGuard::create(&["python3", "-c", localhost_bypass_script()]) + let guard = SandboxGuard::create(&["--", "python3", "-c", localhost_bypass_script()]) .await .expect("sandbox create with localhost proxy bypass check"); diff --git a/e2e/rust/tests/sandbox_lifecycle.rs b/e2e/rust/tests/sandbox_lifecycle.rs index 79ac9b24c3..4caad87371 100644 --- a/e2e/rust/tests/sandbox_lifecycle.rs +++ b/e2e/rust/tests/sandbox_lifecycle.rs @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +#![cfg(feature = "e2e")] + use std::process::Stdio; use std::time::Duration; diff --git a/mise.toml b/mise.toml index 6f855fcf06..e5b898c13f 100644 --- a/mise.toml +++ b/mise.toml @@ -39,7 +39,8 @@ UV_CACHE_DIR = "{{config_root}}/.cache/uv" RUSTC_WRAPPER = "sccache" SCCACHE_DIR = "{{config_root}}/.cache/sccache" -# Shared build constants (overridable via environment) +# Shared build constants (overridable via environment). +# DOCKER_BUILDKIT enables BuildKit for Docker; ignored by podman. DOCKER_BUILDKIT = "1" [vars] diff --git a/scripts/docker-cleanup.sh b/scripts/docker-cleanup.sh index dec57c1545..9669dd80ba 100755 --- a/scripts/docker-cleanup.sh +++ b/scripts/docker-cleanup.sh @@ -22,6 +22,9 @@ set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/../tasks/scripts/container-engine.sh" + # --------------------------------------------------------------------------- # Options # --------------------------------------------------------------------------- @@ -64,13 +67,10 @@ dry() { echo -e " ${YELLOW}[dry-run]${RESET} $*"; } # --------------------------------------------------------------------------- # Preflight # --------------------------------------------------------------------------- -if ! command -v docker &>/dev/null; then - echo "Error: docker is not installed or not in PATH" >&2 - exit 1 -fi +# container-engine.sh already validates that an engine is available. -if ! docker info &>/dev/null; then - echo "Error: Docker daemon is not running" >&2 +if ! ce info &>/dev/null; then + echo "Error: Container engine (${CONTAINER_ENGINE}) is not running" >&2 exit 1 fi @@ -125,8 +125,8 @@ should_keep_image() { # --------------------------------------------------------------------------- # Snapshot disk usage before cleanup # --------------------------------------------------------------------------- -info "Current Docker disk usage" -docker system df +info "Current disk usage (${CONTAINER_ENGINE})" +ce system df echo # --------------------------------------------------------------------------- @@ -156,13 +156,13 @@ TOTAL_VOLUMES_REMOVED=0 # --------------------------------------------------------------------------- info "Removing dangling images..." -dangling_count=$(docker images --filter "dangling=true" -q | wc -l | tr -d ' ') +dangling_count=$(ce images --filter "dangling=true" -q | wc -l | tr -d ' ') if [[ "$dangling_count" -gt 0 ]]; then note "Found $dangling_count dangling image(s)" if [[ "$DRY_RUN" == true ]]; then dry "Would remove $dangling_count dangling image(s)" else - docker image prune -f | tail -1 + ce image prune -f | tail -1 TOTAL_IMAGES_REMOVED=$((TOTAL_IMAGES_REMOVED + dangling_count)) fi else @@ -177,7 +177,7 @@ info "Removing stale tagged images..." # Collect image IDs that are directly used by running containers so we never # touch them regardless of tag matching. -running_image_ids=$(docker ps -q 2>/dev/null | xargs -r docker inspect --format '{{.Image}}' 2>/dev/null | sort -u) +running_image_ids=$(ce ps -q 2>/dev/null | xargs -r ce inspect --format '{{.Image}}' 2>/dev/null | sort -u) stale_images=() while IFS=$'\t' read -r repo tag id; do @@ -192,7 +192,7 @@ while IFS=$'\t' read -r repo tag id; do if ! should_keep_image "$repo"; then stale_images+=("${repo}:${tag}") fi -done < <(docker images --format '{{.Repository}}\t{{.Tag}}\t{{.ID}}') +done < <(ce images --format '{{.Repository}}\t{{.Tag}}\t{{.ID}}') if [[ ${#stale_images[@]} -gt 0 ]]; then for img in "${stale_images[@]}"; do @@ -200,7 +200,7 @@ if [[ ${#stale_images[@]} -gt 0 ]]; then dry "Would remove $img" else note "Removing $img" - docker rmi "$img" >/dev/null 2>&1 || warn "Could not remove $img (may share layers)" + ce rmi "$img" >/dev/null 2>&1 || warn "Could not remove $img (may share layers)" TOTAL_IMAGES_REMOVED=$((TOTAL_IMAGES_REMOVED + 1)) fi done @@ -215,8 +215,8 @@ echo info "Removing unused volumes..." # Identify volumes in use by running containers -in_use_volumes=$(docker ps -q 2>/dev/null \ - | xargs -r docker inspect --format '{{range .Mounts}}{{.Name}} {{end}}' 2>/dev/null \ +in_use_volumes=$(ce ps -q 2>/dev/null \ + | xargs -r ce inspect --format '{{range .Mounts}}{{.Name}} {{end}}' 2>/dev/null \ | tr ' ' '\n' | sort -u | grep -v '^$') unused_volumes=() @@ -225,7 +225,7 @@ while read -r vol; do if ! echo "$in_use_volumes" | grep -qx "$vol" 2>/dev/null; then unused_volumes+=("$vol") fi -done < <(docker volume ls -q) +done < <(ce volume ls -q) if [[ ${#unused_volumes[@]} -gt 0 ]]; then note "Found ${#unused_volumes[@]} unused volume(s)" @@ -233,7 +233,7 @@ if [[ ${#unused_volumes[@]} -gt 0 ]]; then if [[ "$DRY_RUN" == true ]]; then dry "Would remove volume $vol" else - docker volume rm "$vol" >/dev/null 2>&1 || warn "Could not remove volume $vol" + ce volume rm "$vol" >/dev/null 2>&1 || warn "Could not remove volume $vol" TOTAL_VOLUMES_REMOVED=$((TOTAL_VOLUMES_REMOVED + 1)) fi done @@ -249,11 +249,11 @@ if [[ "$SKIP_CACHE" == true ]]; then info "Skipping build cache prune (--skip-cache)" else info "Pruning build cache..." - cache_size=$(docker system df --format '{{.Size}}' 2>/dev/null | tail -1) + cache_size=$(ce system df --format '{{.Size}}' 2>/dev/null | tail -1) if [[ "$DRY_RUN" == true ]]; then dry "Would prune build cache (current size: ${cache_size:-unknown})" else - docker builder prune -af 2>&1 | tail -1 + ce_builder_prune -af 2>&1 | tail -1 fi fi echo @@ -261,11 +261,11 @@ echo # --------------------------------------------------------------------------- # Step 5: Clean up any newly-dangling images left after tagged image removal # --------------------------------------------------------------------------- -remaining_dangling=$(docker images --filter "dangling=true" -q | wc -l | tr -d ' ') +remaining_dangling=$(ce images --filter "dangling=true" -q | wc -l | tr -d ' ') if [[ "$remaining_dangling" -gt 0 ]]; then info "Cleaning up $remaining_dangling residual dangling image(s)..." if [[ "$DRY_RUN" != true ]]; then - docker image prune -f >/dev/null 2>&1 + ce image prune -f >/dev/null 2>&1 TOTAL_IMAGES_REMOVED=$((TOTAL_IMAGES_REMOVED + remaining_dangling)) fi echo @@ -279,9 +279,9 @@ echo if [[ "$DRY_RUN" == true ]]; then warn "Dry run -- no changes were made. Re-run without --dry-run to apply." else - docker system df + ce system df fi echo info "Remaining images:" -docker images --format "table {{.Repository}}\t{{.Tag}}\t{{.Size}}" +ce images --format "table {{.Repository}}\t{{.Tag}}\t{{.Size}}" diff --git a/scripts/remote-deploy.sh b/scripts/remote-deploy.sh index 37a0dca858..8b859b688f 100755 --- a/scripts/remote-deploy.sh +++ b/scripts/remote-deploy.sh @@ -215,8 +215,8 @@ echo "==> Installing tools via mise..." mise trust --yes mise install --yes -if ! command -v docker >/dev/null 2>&1; then - echo "ERROR: Docker is not installed on the remote host." >&2 +if ! command -v podman >/dev/null 2>&1 && ! command -v docker >/dev/null 2>&1; then + echo "ERROR: Neither podman nor docker is installed on the remote host." >&2 exit 1 fi @@ -232,7 +232,7 @@ install -m 0755 target/release/openshell scripts/bin/openshell # Prevent a stale repo-local .env from changing the deployment unexpectedly. rm -f .env -echo "==> Building Docker images (tag=${IMAGE_TAG})..." +echo "==> Building container images (tag=${IMAGE_TAG})..." export OPENSHELL_CARGO_VERSION="${CARGO_VERSION}" export IMAGE_TAG mise exec -- tasks/scripts/docker-build-image.sh cluster diff --git a/tasks/docker.toml b/tasks/docker.toml index f05c8aab15..323b47e13a 100644 --- a/tasks/docker.toml +++ b/tasks/docker.toml @@ -8,6 +8,8 @@ description = "Build all Docker images" depends = [ "build:docker:gateway", "build:docker:cluster", + "build:docker:supervisor", + "build:docker:supervisor-sideload", ] hide = true @@ -22,10 +24,15 @@ run = "tasks/scripts/docker-build-image.sh gateway" hide = true ["build:docker:supervisor"] -description = "Build the supervisor Docker image" +description = "Build the standalone supervisor Docker image (Ubuntu-based, for K8s pods)" run = "tasks/scripts/docker-build-image.sh supervisor" hide = true +["build:docker:supervisor-sideload"] +description = "Build the supervisor sideload image (FROM scratch, for Podman image-volume mount)" +run = "tasks/scripts/docker-build-image.sh supervisor-output" +hide = true + ["build:docker:cluster"] description = "Build the k3s cluster image (component images pulled at runtime from registry)" run = "tasks/scripts/docker-build-image.sh cluster" @@ -36,6 +43,16 @@ description = "Alias for build:docker:gateway" depends = ["build:docker:gateway"] hide = true +["docker:build:supervisor"] +description = "Alias for build:docker:supervisor" +depends = ["build:docker:supervisor"] +hide = true + +["docker:build:supervisor-sideload"] +description = "Alias for build:docker:supervisor-sideload" +depends = ["build:docker:supervisor-sideload"] +hide = true + ["docker:build:cluster"] description = "Alias for build:docker:cluster" depends = ["build:docker:cluster"] @@ -46,6 +63,11 @@ description = "Build multi-arch cluster image and push to a registry" run = "tasks/scripts/docker-publish-multiarch.sh" hide = true +["docker:build:cluster:multiarch"] +description = "Alias for build:docker:cluster:multiarch" +depends = ["build:docker:cluster:multiarch"] +hide = true + ["docker:cleanup"] description = "Remove stale images, volumes, and build cache not used by the current cluster" run = "scripts/docker-cleanup.sh --force" diff --git a/tasks/python.toml b/tasks/python.toml index 1e53da7fbe..a5e1258840 100644 --- a/tasks/python.toml +++ b/tasks/python.toml @@ -143,6 +143,8 @@ run = """ #!/usr/bin/env bash set -euo pipefail +source tasks/scripts/container-engine.sh + sha256_16() { if command -v sha256sum >/dev/null 2>&1; then sha256sum "$1" | awk '{print substr($1, 1, 16)}' @@ -172,7 +174,7 @@ CARGO_TARGET_CACHE_SCOPE=$(printf '%s' "$CACHE_SCOPE_INPUT" | sha256_16_stdin) mkdir -p target/wheels -docker build \ +ce build \ -f deploy/docker/Dockerfile.python-wheels-macos \ --target wheels \ --build-arg "OSXCROSS_IMAGE=${OSXCROSS_IMAGE_REF}" \ diff --git a/tasks/scripts/cluster-bootstrap.sh b/tasks/scripts/cluster-bootstrap.sh index def2429b88..d75bbc0582 100755 --- a/tasks/scripts/cluster-bootstrap.sh +++ b/tasks/scripts/cluster-bootstrap.sh @@ -5,6 +5,9 @@ set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/container-engine.sh" + # Normalize cluster name: lowercase, replace invalid chars with hyphens normalize_name() { echo "$1" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g' | sed 's/--*/-/g' | sed 's/^-//;s/-$//' @@ -142,28 +145,35 @@ wait_for_registry_ready() { } ensure_local_registry() { - if docker inspect "${LOCAL_REGISTRY_CONTAINER}" >/dev/null 2>&1; then + if ce inspect "${LOCAL_REGISTRY_CONTAINER}" >/dev/null 2>&1; then local proxy_remote_url - proxy_remote_url=$(docker inspect "${LOCAL_REGISTRY_CONTAINER}" --format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null | awk -F= '/^REGISTRY_PROXY_REMOTEURL=/{print $2; exit}' || true) + proxy_remote_url=$(ce inspect "${LOCAL_REGISTRY_CONTAINER}" --format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null | awk -F= '/^REGISTRY_PROXY_REMOTEURL=/{print $2; exit}' || true) if [ -n "${proxy_remote_url}" ]; then - docker rm -f "${LOCAL_REGISTRY_CONTAINER}" >/dev/null 2>&1 || true + ce rm -f "${LOCAL_REGISTRY_CONTAINER}" >/dev/null 2>&1 || true fi fi - if ! docker inspect "${LOCAL_REGISTRY_CONTAINER}" >/dev/null 2>&1; then - docker run -d --restart=always --name "${LOCAL_REGISTRY_CONTAINER}" -p 5000:5000 registry:2 >/dev/null + if ! ce inspect "${LOCAL_REGISTRY_CONTAINER}" >/dev/null 2>&1; then + # --restart=always: on Docker this is managed by the Docker daemon and + # survives reboots when the daemon is enabled at boot. On rootless Podman + # it is handled by conmon within the current login session; it does NOT + # survive user logout or reboot without a systemd user unit. This is + # acceptable here because ensure_local_registry() recreates the container + # on every cluster bootstrap invocation if it is missing. + ce run -d --restart=always --name "${LOCAL_REGISTRY_CONTAINER}" -p 5000:5000 registry:2 >/dev/null else - if ! docker ps --filter "name=^${LOCAL_REGISTRY_CONTAINER}$" --filter "status=running" -q | grep -q .; then - docker start "${LOCAL_REGISTRY_CONTAINER}" >/dev/null + if ! ce ps --filter "name=^${LOCAL_REGISTRY_CONTAINER}$" --filter "status=running" -q | grep -q .; then + ce start "${LOCAL_REGISTRY_CONTAINER}" >/dev/null fi - port_map=$(docker port "${LOCAL_REGISTRY_CONTAINER}" 5000/tcp 2>/dev/null || true) + port_map=$(ce port "${LOCAL_REGISTRY_CONTAINER}" 5000/tcp 2>/dev/null || true) case "${port_map}" in *:5000*) ;; *) - docker rm -f "${LOCAL_REGISTRY_CONTAINER}" >/dev/null 2>&1 || true - docker run -d --restart=always --name "${LOCAL_REGISTRY_CONTAINER}" -p 5000:5000 registry:2 >/dev/null + ce rm -f "${LOCAL_REGISTRY_CONTAINER}" >/dev/null 2>&1 || true + # See --restart=always note above in this function. + ce run -d --restart=always --name "${LOCAL_REGISTRY_CONTAINER}" -p 5000:5000 registry:2 >/dev/null ;; esac fi @@ -177,9 +187,9 @@ ensure_local_registry() { fi echo "Error: local registry is not reachable at ${REGISTRY_HOST}." >&2 - echo " Ensure a registry is running on port 5000 (e.g. docker run -d --name openshell-local-registry -p 5000:5000 registry:2)." >&2 - docker ps -a >&2 || true - docker logs "${LOCAL_REGISTRY_CONTAINER}" >&2 || true + echo " Ensure a registry is running on port 5000 (e.g. ${CONTAINER_ENGINE} run -d --name openshell-local-registry -p 5000:5000 registry:2)." >&2 + ce ps -a >&2 || true + ce logs "${LOCAL_REGISTRY_CONTAINER}" >&2 || true exit 1 } @@ -201,7 +211,7 @@ export IMAGE_REPO_BASE export IMAGE_TAG if [ -n "${CI:-}" ] && [ -n "${CI_REGISTRY:-}" ] && [ -n "${CI_REGISTRY_USER:-}" ] && [ -n "${CI_REGISTRY_PASSWORD:-}" ]; then - printf '%s' "${CI_REGISTRY_PASSWORD}" | docker login -u "${CI_REGISTRY_USER}" --password-stdin "${CI_REGISTRY}" + printf '%s' "${CI_REGISTRY_PASSWORD}" | ce login -u "${CI_REGISTRY_USER}" --password-stdin "${CI_REGISTRY}" export OPENSHELL_REGISTRY_USERNAME=${OPENSHELL_REGISTRY_USERNAME:-${CI_REGISTRY_USER}} export OPENSHELL_REGISTRY_PASSWORD=${OPENSHELL_REGISTRY_PASSWORD:-${CI_REGISTRY_PASSWORD}} fi @@ -214,7 +224,7 @@ CONTAINER_NAME="openshell-cluster-${CLUSTER_NAME}" VOLUME_NAME="openshell-cluster-${CLUSTER_NAME}" if [ "${MODE}" = "fast" ]; then - if docker inspect "${CONTAINER_NAME}" >/dev/null 2>&1 || docker volume inspect "${VOLUME_NAME}" >/dev/null 2>&1; then + if ce inspect "${CONTAINER_NAME}" >/dev/null 2>&1 || ce volume inspect "${VOLUME_NAME}" >/dev/null 2>&1; then echo "Recreating cluster '${CLUSTER_NAME}' from scratch..." openshell gateway destroy --name "${CLUSTER_NAME}" fi @@ -255,7 +265,7 @@ if [ -n "${GATEWAY_HOST:-}" ]; then # (it's a Docker Desktop feature). If the hostname doesn't resolve, # add it via the Docker bridge gateway IP. if ! getent hosts "${GATEWAY_HOST}" >/dev/null 2>&1; then - BRIDGE_IP=$(docker network inspect bridge --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || true) + BRIDGE_IP=$(ce_network_gateway) if [ -n "${BRIDGE_IP}" ]; then echo "Adding /etc/hosts entry: ${BRIDGE_IP} ${GATEWAY_HOST}" echo "${BRIDGE_IP} ${GATEWAY_HOST}" >> /etc/hosts diff --git a/tasks/scripts/cluster-deploy-fast.sh b/tasks/scripts/cluster-deploy-fast.sh index d9359cf4f8..b4d79d4eb1 100755 --- a/tasks/scripts/cluster-deploy-fast.sh +++ b/tasks/scripts/cluster-deploy-fast.sh @@ -5,6 +5,9 @@ set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/container-engine.sh" + # Normalize cluster name: lowercase, replace invalid chars with hyphens normalize_name() { echo "$1" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g' | sed 's/--*/-/g' | sed 's/^-//;s/-$//' @@ -30,7 +33,7 @@ log_duration() { echo "${label} took $((end - start))s" } -if ! docker ps -q --filter "name=^${CONTAINER_NAME}$" --filter "health=healthy" | grep -q .; then +if ! ce ps -q --filter "name=^${CONTAINER_NAME}$" --filter "health=healthy" | grep -q .; then echo "Error: Cluster container '${CONTAINER_NAME}' is not running or not healthy." echo "Start the cluster first with: mise run cluster" exit 1 @@ -38,7 +41,7 @@ fi # Run a command inside the cluster container with KUBECONFIG pre-configured. cluster_exec() { - docker exec "${CONTAINER_NAME}" sh -c "KUBECONFIG=/etc/rancher/k3s/k3s.yaml $*" + ce exec "${CONTAINER_NAME}" sh -c "KUBECONFIG=/etc/rancher/k3s/k3s.yaml $*" } # Path inside the container where the chart is copied for helm upgrades. @@ -102,7 +105,7 @@ log_duration "Change detection" "${detect_start}" "${detect_end}" # recreated (e.g. via bootstrap). A new container means the k3s state is # fresh and all images must be rebuilt and pushed regardless of source # fingerprints. -current_container_id=$(docker inspect --format '{{.Id}}' "${CONTAINER_NAME}" 2>/dev/null || true) +current_container_id=$(ce inspect --format '{{.Id}}' "${CONTAINER_NAME}" 2>/dev/null || true) if [[ -f "${DEPLOY_FAST_STATE_FILE}" ]]; then while IFS='=' read -r key value; do @@ -315,11 +318,11 @@ if [[ "${build_supervisor}" == "1" ]]; then # Detect the cluster container's architecture so we cross-compile correctly. # Container objects lack an Architecture field (the Go template emits a # stray newline before erroring), so inspect the container's *image* instead. - _cluster_image=$(docker inspect --format '{{.Config.Image}}' "${CONTAINER_NAME}" 2>/dev/null) - CLUSTER_ARCH=$(docker image inspect --format '{{.Architecture}}' "${_cluster_image}" 2>/dev/null || echo "amd64") + _cluster_image=$(ce inspect --format '{{.Config.Image}}' "${CONTAINER_NAME}" 2>/dev/null) + CLUSTER_ARCH=$(ce image inspect --format '{{.Architecture}}' "${_cluster_image}" 2>/dev/null || echo "amd64") - # Detect the host (build) architecture in Docker's naming convention. - HOST_ARCH=$(docker info --format '{{.Architecture}}' 2>/dev/null || echo "amd64") + # Detect the host (build) architecture in the container engine's naming convention. + HOST_ARCH=$(ce_info_arch) # Normalize: Docker reports "aarch64" on ARM hosts but uses "arm64" elsewhere. case "${HOST_ARCH}" in aarch64) HOST_ARCH=arm64 ;; @@ -353,10 +356,10 @@ if [[ "${build_supervisor}" == "1" ]]; then tasks/scripts/docker-build-image.sh supervisor-output # Copy the built binary into the running k3s container - docker exec "${CONTAINER_NAME}" mkdir -p /opt/openshell/bin - docker cp "${SUPERVISOR_BUILD_DIR}/openshell-sandbox" \ + ce exec "${CONTAINER_NAME}" mkdir -p /opt/openshell/bin + ce cp "${SUPERVISOR_BUILD_DIR}/openshell-sandbox" \ "${CONTAINER_NAME}:/opt/openshell/bin/openshell-sandbox" - docker exec "${CONTAINER_NAME}" chmod 755 /opt/openshell/bin/openshell-sandbox + ce exec "${CONTAINER_NAME}" chmod 755 /opt/openshell/bin/openshell-sandbox built_components+=("supervisor") supervisor_end=$(date +%s) @@ -370,7 +373,7 @@ log_duration "Builds" "${build_start}" "${build_end}" declare -a pushed_images=() if [[ "${build_gateway}" == "1" ]]; then - docker tag "openshell/gateway:${IMAGE_TAG}" "${IMAGE_REPO_BASE}/gateway:${IMAGE_TAG}" 2>/dev/null || true + ce tag "openshell/gateway:${IMAGE_TAG}" "${IMAGE_REPO_BASE}/gateway:${IMAGE_TAG}" 2>/dev/null || true pushed_images+=("${IMAGE_REPO_BASE}/gateway:${IMAGE_TAG}") built_components+=("gateway") fi @@ -379,7 +382,7 @@ if [[ "${#pushed_images[@]}" -gt 0 ]]; then push_start=$(date +%s) echo "Pushing updated images to local registry..." for image_ref in "${pushed_images[@]}"; do - docker push "${image_ref}" + ce push "${image_ref}" done push_end=$(date +%s) log_duration "Image push" "${push_start}" "${push_end}" @@ -389,7 +392,7 @@ fi # the updated image from the registry. if [[ "${build_gateway}" == "1" ]]; then echo "Evicting stale gateway image from k3s..." - docker exec "${CONTAINER_NAME}" crictl rmi "${IMAGE_REPO_BASE}/gateway:${IMAGE_TAG}" >/dev/null 2>&1 || true + ce exec "${CONTAINER_NAME}" crictl rmi "${IMAGE_REPO_BASE}/gateway:${IMAGE_TAG}" >/dev/null 2>&1 || true fi if [[ "${needs_helm_upgrade}" == "1" ]]; then @@ -401,8 +404,8 @@ if [[ "${needs_helm_upgrade}" == "1" ]]; then fi # Copy the local chart source into the container so helm can read it. - docker exec "${CONTAINER_NAME}" rm -rf "${CONTAINER_CHART_DIR}" - docker cp deploy/helm/openshell "${CONTAINER_NAME}:${CONTAINER_CHART_DIR}" + ce exec "${CONTAINER_NAME}" rm -rf "${CONTAINER_CHART_DIR}" + ce cp deploy/helm/openshell "${CONTAINER_NAME}:${CONTAINER_CHART_DIR}" # grpcEndpoint must be explicitly set to https:// because the chart always # terminates mTLS (there is no server.tls.enabled toggle). Without this, diff --git a/tasks/scripts/cluster-push-component.sh b/tasks/scripts/cluster-push-component.sh index a59192efb6..f94f839231 100755 --- a/tasks/scripts/cluster-push-component.sh +++ b/tasks/scripts/cluster-push-component.sh @@ -5,6 +5,9 @@ set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/container-engine.sh" + component=${1:-} if [ -z "${component}" ]; then echo "usage: $0 " >&2 @@ -41,7 +44,7 @@ source_candidates=( resolved_source_image="" for candidate in "${source_candidates[@]}"; do - if docker image inspect "${candidate}" >/dev/null 2>&1; then + if ce image inspect "${candidate}" >/dev/null 2>&1; then resolved_source_image="${candidate}" break fi @@ -53,12 +56,12 @@ if [ -z "${resolved_source_image}" ]; then resolved_source_image="openshell/${component}:${IMAGE_TAG}" fi -docker tag "${resolved_source_image}" "${TARGET_IMAGE}" -docker push "${TARGET_IMAGE}" +ce tag "${resolved_source_image}" "${TARGET_IMAGE}" +ce push "${TARGET_IMAGE}" # Evict the stale image from k3s's containerd cache so new pods pull the # updated image. Without this, k3s uses its cached copy (imagePullPolicy # defaults to IfNotPresent for non-:latest tags) and pods run stale code. -if docker ps -q --filter "name=${CONTAINER_NAME}" | grep -q .; then - docker exec "${CONTAINER_NAME}" crictl rmi "${TARGET_IMAGE}" >/dev/null 2>&1 || true +if ce ps -q --filter "name=${CONTAINER_NAME}" | grep -q .; then + ce exec "${CONTAINER_NAME}" crictl rmi "${TARGET_IMAGE}" >/dev/null 2>&1 || true fi diff --git a/tasks/scripts/cluster.sh b/tasks/scripts/cluster.sh index 6dc6bf3d49..33b8b8e889 100755 --- a/tasks/scripts/cluster.sh +++ b/tasks/scripts/cluster.sh @@ -8,6 +8,9 @@ set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/container-engine.sh" + # Normalize cluster name: lowercase, replace invalid chars with hyphens normalize_name() { echo "$1" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g' | sed 's/--*/-/g' | sed 's/^-//;s/-$//' @@ -17,13 +20,13 @@ CLUSTER_NAME=${CLUSTER_NAME:-$(basename "$PWD")} CLUSTER_NAME=$(normalize_name "${CLUSTER_NAME}") CONTAINER_NAME="openshell-cluster-${CLUSTER_NAME}" -if ! docker ps -q --filter "name=${CONTAINER_NAME}" | grep -q .; then +if ! ce ps -q --filter "name=${CONTAINER_NAME}" | grep -q .; then echo "No running cluster found. Bootstrapping..." exec tasks/scripts/cluster-bootstrap.sh fast fi # Container is running but not healthy — tear it down and re-bootstrap. -if ! docker ps -q --filter "name=^${CONTAINER_NAME}$" --filter "health=healthy" | grep -q .; then +if ! ce ps -q --filter "name=^${CONTAINER_NAME}$" --filter "health=healthy" | grep -q .; then echo "Cluster container '${CONTAINER_NAME}' is running but not healthy. Recreating..." exec tasks/scripts/cluster-bootstrap.sh fast fi diff --git a/tasks/scripts/container-engine.sh b/tasks/scripts/container-engine.sh new file mode 100755 index 0000000000..2cb0dd7ff6 --- /dev/null +++ b/tasks/scripts/container-engine.sh @@ -0,0 +1,324 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Shared container engine detection and abstraction layer. +# +# Source this file in any script that needs to run container commands: +# SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# source "${SCRIPT_DIR}/container-engine.sh" # or adjust path accordingly +# +# After sourcing, use these instead of bare `docker` / `podman`: +# ce [args...] — run container engine command +# ce_build [args...] — container image build (handles buildx differences) +# ce_is_podman / ce_is_docker — check which engine is active +# ce_info_arch — host architecture (handles format differences) +# ce_network_gateway [network] — default network gateway IP +# ce_builder_prune [args...] — prune build cache +# ce_buildx_inspect [args...] — inspect buildx builder (no-op for podman) +# ce_build_multiarch — multi-arch build + push workflow +# +# Override the auto-detected engine by setting CONTAINER_ENGINE=docker or +# CONTAINER_ENGINE=podman before sourcing. +# Suppress the detection log line with CONTAINER_ENGINE_QUIET=1 (useful in +# CI pipelines or scripts that source this file multiple times in a pipeline). + +# Guard against double-sourcing. +if [[ -n "${_CONTAINER_ENGINE_LOADED:-}" ]]; then + return 0 +fi +_CONTAINER_ENGINE_LOADED=1 + +# --------------------------------------------------------------------------- +# Detection +# --------------------------------------------------------------------------- + +_detect_container_engine() { + # Honour explicit override. + if [[ -n "${CONTAINER_ENGINE:-}" ]]; then + if ! command -v "${CONTAINER_ENGINE}" >/dev/null 2>&1; then + echo "Error: CONTAINER_ENGINE=${CONTAINER_ENGINE} is not installed or not in PATH" >&2 + exit 1 + fi + _CE_EXPLICIT_OVERRIDE=1 + return + fi + + # Prefer podman when available. + if command -v podman >/dev/null 2>&1; then + CONTAINER_ENGINE=podman + return + fi + + # Fall back to docker — but detect the podman-masquerading-as-docker shim + # shipped by some distros (e.g. Fedora, RHEL). + if command -v docker >/dev/null 2>&1; then + if docker --version 2>/dev/null | grep -qi podman; then + CONTAINER_ENGINE=podman + else + CONTAINER_ENGINE=docker + fi + return + fi + + echo "Error: neither podman nor docker is installed." >&2 + echo " Install one of them and try again." >&2 + exit 1 +} + +_detect_container_engine + +# The actual binary to invoke — usually equals CONTAINER_ENGINE, but when +# podman is detected via the docker shim we still call `docker` (the shim +# execs podman internally). +_CE_BIN="${CONTAINER_ENGINE}" +if [[ "${CONTAINER_ENGINE}" == "podman" ]] && ! command -v podman >/dev/null 2>&1; then + # podman detected through docker shim; call docker (which execs podman). + _CE_BIN=docker +fi + +# --------------------------------------------------------------------------- +# Core helpers +# --------------------------------------------------------------------------- + +# Run the container engine with arbitrary arguments. +ce() { + "${_CE_BIN}" "$@" +} + +ce_is_podman() { + [[ "${CONTAINER_ENGINE}" == "podman" ]] +} + +ce_is_docker() { + [[ "${CONTAINER_ENGINE}" == "docker" ]] +} + +# --------------------------------------------------------------------------- +# ce_build — abstraction over `docker buildx build` / `podman build` +# +# Accepts the same flags as `docker buildx build`. For podman the function: +# - Strips --load (podman loads locally by default) +# - Strips --provenance (podman doesn't generate provenance attestations) +# - Strips --builder (podman has no builder concept) +# - Converts --push to a post-build `podman push` (podman build has no --push) +# +# All other flags (--platform, --build-arg, --cache-from, --cache-to, +# --output, -t, -f, --target, etc.) are passed through as-is since podman +# build supports them. +# --------------------------------------------------------------------------- +ce_build() { + if ce_is_docker; then + "${_CE_BIN}" buildx build "$@" + return + fi + + # Podman path — filter out unsupported flags. + local args=() + local push_after=false + local image_tags=() + + while [[ $# -gt 0 ]]; do + case "$1" in + --load) + # Podman loads locally by default; skip. + shift + ;; + --push) + push_after=true + shift + ;; + --provenance|--provenance=*) + # Podman doesn't support provenance attestations; skip. + shift + ;; + --builder|--builder=*) + # Podman has no builder concept; skip. + if [[ "$1" == "--builder" ]]; then + shift 2 # skip --builder + else + shift # skip --builder= + fi + ;; + -t|--tag) + image_tags+=("$2") + args+=("$1" "$2") + shift 2 + ;; + -t=*|--tag=*) + image_tags+=("${1#*=}") + args+=("$1") + shift + ;; + *) + args+=("$1") + shift + ;; + esac + done + + "${_CE_BIN}" build "${args[@]}" + + if [[ "${push_after}" == "true" ]]; then + for tag in "${image_tags[@]}"; do + "${_CE_BIN}" push "${tag}" + done + fi +} + +# --------------------------------------------------------------------------- +# ce_info_arch — host architecture reported by the container engine. +# +# Docker: docker info --format '{{.Architecture}}' +# Podman: podman info --format '{{.Host.Arch}}' +# --------------------------------------------------------------------------- +ce_info_arch() { + if ce_is_docker; then + "${_CE_BIN}" info --format '{{.Architecture}}' 2>/dev/null || echo "amd64" + else + "${_CE_BIN}" info --format '{{.Host.Arch}}' 2>/dev/null || echo "amd64" + fi +} + +# --------------------------------------------------------------------------- +# ce_network_gateway — default network gateway IP. +# +# Docker: docker network inspect bridge --format '{{(index .IPAM.Config 0).Gateway}}' +# Podman: podman network inspect podman --format '{{(index .Subnets 0).Gateway}}' +# +# Accepts an optional network name override; defaults to the engine's default. +# --------------------------------------------------------------------------- +ce_network_gateway() { + local network="${1:-}" + if ce_is_docker; then + network="${network:-bridge}" + "${_CE_BIN}" network inspect "${network}" --format '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || true + else + network="${network:-podman}" + "${_CE_BIN}" network inspect "${network}" --format '{{(index .Subnets 0).Gateway}}' 2>/dev/null || true + fi +} + +# --------------------------------------------------------------------------- +# ce_builder_prune — prune build cache. +# +# Docker: docker builder prune -af +# Podman: podman system reset is too aggressive; use buildah prune or image prune. +# --------------------------------------------------------------------------- +ce_builder_prune() { + if ce_is_docker; then + "${_CE_BIN}" builder prune "$@" + else + # Podman doesn't have `builder prune`. `podman image prune` removes + # dangling build layers. `buildah prune` is closest but may not be + # installed. Fall back gracefully. + if command -v buildah >/dev/null 2>&1; then + buildah prune "$@" + else + "${_CE_BIN}" image prune "$@" + fi + fi +} + +# --------------------------------------------------------------------------- +# ce_buildx_inspect — inspect a buildx builder. +# +# Docker: docker buildx inspect [name] +# Podman: returns a synthetic "podman" driver response to satisfy callers that +# check for "Driver: docker-container". +# --------------------------------------------------------------------------- +ce_buildx_inspect() { + if ce_is_docker; then + "${_CE_BIN}" buildx inspect "$@" + else + # Podman doesn't have real buildx builders. Emit a minimal response so + # callers that grep for "Driver:" get a predictable answer. + echo "Name: default" + echo "Driver: podman" + fi +} + +# --------------------------------------------------------------------------- +# ce_context_name — current context/connection name. +# +# Docker: docker context inspect --format '{{.Name}}' +# Podman: always "default" +# --------------------------------------------------------------------------- +ce_context_name() { + if ce_is_docker; then + "${_CE_BIN}" context inspect --format '{{.Name}}' 2>/dev/null || echo "default" + else + echo "default" + fi +} + +# --------------------------------------------------------------------------- +# ce_imagetools_create — create/re-tag a multi-arch manifest. +# +# Docker: docker buildx imagetools create -t +# Podman: no direct equivalent — the caller should use podman manifest +# workflows instead. This helper exists so scripts can call it +# without engine checks; for podman it falls back to tag + push. +# --------------------------------------------------------------------------- +ce_imagetools_create() { + if ce_is_docker; then + "${_CE_BIN}" buildx imagetools create "$@" + return + fi + + # Podman fallback: parse -t and the trailing source image, then + # use skopeo or podman tag. This is a best-effort shim for simple + # re-tagging; full multi-arch manifest manipulation should use the + # podman-native code path in docker-publish-multiarch.sh. + # + # Argument parsing uses a sentinel ("__next__") to capture the value + # that follows a two-token -t / --tag flag. --prefer-index is accepted + # and silently ignored (the Docker path passes it through to buildx; + # the Podman path has no equivalent concept). + local new_tag="" source_image="" + for arg in "$@"; do + case "${arg}" in + -t|--tag) + new_tag="__next__" + continue + ;; + --prefer-index|--prefer-index=*) + # No podman equivalent; accepted and ignored for call-site compatibility. + continue + ;; + esac + if [[ "${new_tag}" == "__next__" ]]; then + new_tag="${arg}" + else + source_image="${arg}" + fi + done + + if [[ -n "${new_tag}" && -n "${source_image}" ]]; then + if command -v skopeo >/dev/null 2>&1; then + skopeo copy --all "docker://${source_image}" "docker://${new_tag}" + else + "${_CE_BIN}" tag "${source_image}" "${new_tag}" + "${_CE_BIN}" push "${new_tag}" + fi + fi +} + +# --------------------------------------------------------------------------- +# Log the detected engine so developers always know which tool is active. +# Emitted once per script invocation (the double-source guard at the top +# prevents repeated output when scripts source each other). +# Suppress with CONTAINER_ENGINE_QUIET=1 for CI or non-interactive use. +# --------------------------------------------------------------------------- +_ce_log_detected() { + if [[ "${CONTAINER_ENGINE_QUIET:-}" == "1" ]]; then + return + fi + if [[ -n "${_CE_EXPLICIT_OVERRIDE:-}" ]]; then + echo "[container-engine] using ${CONTAINER_ENGINE} (set via CONTAINER_ENGINE env)" >&2 + else + echo "[container-engine] auto-detected: ${CONTAINER_ENGINE} (override with CONTAINER_ENGINE=docker|podman)" >&2 + fi +} +_ce_log_detected diff --git a/tasks/scripts/docker-build-ci.sh b/tasks/scripts/docker-build-ci.sh index d10c6f8bea..da5f618bf8 100755 --- a/tasks/scripts/docker-build-ci.sh +++ b/tasks/scripts/docker-build-ci.sh @@ -8,6 +8,9 @@ set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/container-engine.sh" + OUTPUT_ARGS=(--load) if [[ "${DOCKER_PUSH:-}" == "1" ]]; then OUTPUT_ARGS=(--push) @@ -15,7 +18,7 @@ elif [[ "${DOCKER_PLATFORM:-}" == *","* ]]; then OUTPUT_ARGS=(--push) fi -exec docker buildx build \ +exec ce_build \ ${DOCKER_BUILDER:+--builder ${DOCKER_BUILDER}} \ ${DOCKER_PLATFORM:+--platform ${DOCKER_PLATFORM}} \ -f deploy/docker/Dockerfile.ci \ diff --git a/tasks/scripts/docker-build-image.sh b/tasks/scripts/docker-build-image.sh index e429d03396..d6cd4da295 100755 --- a/tasks/scripts/docker-build-image.sh +++ b/tasks/scripts/docker-build-image.sh @@ -5,37 +5,40 @@ set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/container-engine.sh" + sha256_16() { - if command -v sha256sum >/dev/null 2>&1; then - sha256sum "$1" | awk '{print substr($1, 1, 16)}' - else - shasum -a 256 "$1" | awk '{print substr($1, 1, 16)}' - fi + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print substr($1, 1, 16)}' + else + shasum -a 256 "$1" | awk '{print substr($1, 1, 16)}' + fi } sha256_16_stdin() { - if command -v sha256sum >/dev/null 2>&1; then - sha256sum | awk '{print substr($1, 1, 16)}' - else - shasum -a 256 | awk '{print substr($1, 1, 16)}' - fi + if command -v sha256sum >/dev/null 2>&1; then + sha256sum | awk '{print substr($1, 1, 16)}' + else + shasum -a 256 | awk '{print substr($1, 1, 16)}' + fi } detect_rust_scope() { - local dockerfile="$1" - local rust_from - rust_from=$(grep -E '^FROM --platform=\$BUILDPLATFORM rust:[^ ]+' "$dockerfile" | head -n1 | sed -E 's/^FROM --platform=\$BUILDPLATFORM rust:([^ ]+).*/\1/' || true) - if [[ -n "${rust_from}" ]]; then - echo "rust-${rust_from}" - return - fi - - if grep -q "rustup.rs" "$dockerfile"; then - echo "rustup-stable" - return - fi - - echo "no-rust" + local dockerfile="$1" + local rust_from + rust_from=$(grep -E '^FROM --platform=\$BUILDPLATFORM rust:[^ ]+' "$dockerfile" | head -n1 | sed -E 's/^FROM --platform=\$BUILDPLATFORM rust:([^ ]+).*/\1/' || true) + if [[ -n "${rust_from}" ]]; then + echo "rust-${rust_from}" + return + fi + + if grep -q "rustup.rs" "$dockerfile"; then + echo "rustup-stable" + return + fi + + echo "no-rust" } TARGET=${1:?"Usage: docker-build-image.sh [extra-args...]"} @@ -43,8 +46,8 @@ shift DOCKERFILE="deploy/docker/Dockerfile.images" if [[ ! -f "${DOCKERFILE}" ]]; then - echo "Error: Dockerfile not found: ${DOCKERFILE}" >&2 - exit 1 + echo "Error: Dockerfile not found: ${DOCKERFILE}" >&2 + exit 1 fi IS_FINAL_IMAGE=0 @@ -70,6 +73,8 @@ case "${TARGET}" in DOCKER_TARGET="supervisor-builder" ;; supervisor-output) + IS_FINAL_IMAGE=1 + IMAGE_NAME="openshell/supervisor" DOCKER_TARGET="supervisor-output" ;; *) @@ -79,7 +84,7 @@ case "${TARGET}" in esac if [[ -n "${IMAGE_REGISTRY:-}" && "${IS_FINAL_IMAGE}" == "1" ]]; then - IMAGE_NAME="${IMAGE_REGISTRY}/${IMAGE_NAME#openshell/}" + IMAGE_NAME="${IMAGE_REGISTRY}/${IMAGE_NAME#openshell/}" fi IMAGE_TAG=${IMAGE_TAG:-dev} @@ -88,36 +93,40 @@ CACHE_PATH="${DOCKER_BUILD_CACHE_DIR}/images" mkdir -p "${CACHE_PATH}" BUILDER_ARGS=() -if [[ -n "${DOCKER_BUILDER:-}" ]]; then - BUILDER_ARGS=(--builder "${DOCKER_BUILDER}") -elif [[ -z "${DOCKER_PLATFORM:-}" && -z "${CI:-}" ]]; then - _ctx=$(docker context inspect --format '{{.Name}}' 2>/dev/null || echo default) - BUILDER_ARGS=(--builder "${_ctx}") +if ce_is_docker; then + if [[ -n "${DOCKER_BUILDER:-}" ]]; then + BUILDER_ARGS=(--builder "${DOCKER_BUILDER}") + elif [[ -z "${DOCKER_PLATFORM:-}" && -z "${CI:-}" ]]; then + _ctx=$(ce_context_name) + BUILDER_ARGS=(--builder "${_ctx}") + fi fi CACHE_ARGS=() if [[ -z "${CI:-}" ]]; then - if docker buildx inspect ${BUILDER_ARGS[@]+"${BUILDER_ARGS[@]}"} 2>/dev/null | grep -q "Driver: docker-container"; then - CACHE_ARGS=( - --cache-from "type=local,src=${CACHE_PATH}" - --cache-to "type=local,dest=${CACHE_PATH},mode=max" - ) - fi + if ce_is_docker; then + if ce_buildx_inspect ${BUILDER_ARGS[@]+"${BUILDER_ARGS[@]}"} 2>/dev/null | grep -q "Driver: docker-container"; then + CACHE_ARGS=( + --cache-from "type=local,src=${CACHE_PATH}" + --cache-to "type=local,dest=${CACHE_PATH},mode=max" + ) + fi + fi fi SCCACHE_ARGS=() if [[ -n "${SCCACHE_MEMCACHED_ENDPOINT:-}" ]]; then - SCCACHE_ARGS=(--build-arg "SCCACHE_MEMCACHED_ENDPOINT=${SCCACHE_MEMCACHED_ENDPOINT}") + SCCACHE_ARGS=(--build-arg "SCCACHE_MEMCACHED_ENDPOINT=${SCCACHE_MEMCACHED_ENDPOINT}") fi VERSION_ARGS=() if [[ -n "${OPENSHELL_CARGO_VERSION:-}" ]]; then - VERSION_ARGS=(--build-arg "OPENSHELL_CARGO_VERSION=${OPENSHELL_CARGO_VERSION}") + VERSION_ARGS=(--build-arg "OPENSHELL_CARGO_VERSION=${OPENSHELL_CARGO_VERSION}") elif [[ -n "${CI:-}" ]]; then - CARGO_VERSION=$(uv run python tasks/scripts/release.py get-version --cargo 2>/dev/null || true) - if [[ -n "${CARGO_VERSION}" ]]; then - VERSION_ARGS=(--build-arg "OPENSHELL_CARGO_VERSION=${CARGO_VERSION}") - fi + CARGO_VERSION=$(uv run python tasks/scripts/release.py get-version --cargo 2>/dev/null || true) + if [[ -n "${CARGO_VERSION}" ]]; then + VERSION_ARGS=(--build-arg "OPENSHELL_CARGO_VERSION=${CARGO_VERSION}") + fi fi LOCK_HASH=$(sha256_16 Cargo.lock) @@ -127,41 +136,41 @@ CARGO_TARGET_CACHE_SCOPE=$(printf '%s' "${CACHE_SCOPE_INPUT}" | sha256_16_stdin) # The cluster image embeds the packaged Helm chart. if [[ "${TARGET}" == "cluster" ]]; then - mkdir -p deploy/docker/.build/charts - helm package deploy/helm/openshell -d deploy/docker/.build/charts/ >/dev/null + mkdir -p deploy/docker/.build/charts + helm package deploy/helm/openshell -d deploy/docker/.build/charts/ >/dev/null fi K3S_ARGS=() if [[ "${TARGET}" == "cluster" && -n "${K3S_VERSION:-}" ]]; then - K3S_ARGS=(--build-arg "K3S_VERSION=${K3S_VERSION}") + K3S_ARGS=(--build-arg "K3S_VERSION=${K3S_VERSION}") fi # CI builds use codegen-units=1 for maximum optimization; local builds omit # the arg so cargo uses the Cargo.toml default (parallel codegen, fast links). CODEGEN_ARGS=() if [[ -n "${CI:-}" ]]; then - CODEGEN_ARGS=(--build-arg "CARGO_CODEGEN_UNITS=1") + CODEGEN_ARGS=(--build-arg "CARGO_CODEGEN_UNITS=1") fi TAG_ARGS=() if [[ "${IS_FINAL_IMAGE}" == "1" ]]; then - TAG_ARGS=(-t "${IMAGE_NAME}:${IMAGE_TAG}") + TAG_ARGS=(-t "${IMAGE_NAME}:${IMAGE_TAG}") fi OUTPUT_ARGS=() if [[ -n "${DOCKER_OUTPUT:-}" ]]; then - OUTPUT_ARGS=(--output "${DOCKER_OUTPUT}") + OUTPUT_ARGS=(--output "${DOCKER_OUTPUT}") elif [[ "${IS_FINAL_IMAGE}" == "1" ]]; then - if [[ "${DOCKER_PUSH:-}" == "1" ]]; then - OUTPUT_ARGS=(--push) - elif [[ "${DOCKER_PLATFORM:-}" == *","* ]]; then - OUTPUT_ARGS=(--push) - else - OUTPUT_ARGS=(--load) - fi + if [[ "${DOCKER_PUSH:-}" == "1" ]]; then + OUTPUT_ARGS=(--push) + elif [[ "${DOCKER_PLATFORM:-}" == *","* ]]; then + OUTPUT_ARGS=(--push) + else + OUTPUT_ARGS=(--load) + fi else - echo "Error: DOCKER_OUTPUT must be set when building target '${TARGET}'" >&2 - exit 1 + echo "Error: DOCKER_OUTPUT must be set when building target '${TARGET}'" >&2 + exit 1 fi # Default to dev-settings so local builds include test-only settings @@ -170,23 +179,23 @@ EXTRA_CARGO_FEATURES="${EXTRA_CARGO_FEATURES:-openshell-core/dev-settings}" FEATURE_ARGS=() if [[ -n "${EXTRA_CARGO_FEATURES}" ]]; then - FEATURE_ARGS=(--build-arg "EXTRA_CARGO_FEATURES=${EXTRA_CARGO_FEATURES}") + FEATURE_ARGS=(--build-arg "EXTRA_CARGO_FEATURES=${EXTRA_CARGO_FEATURES}") fi -docker buildx build \ - ${BUILDER_ARGS[@]+"${BUILDER_ARGS[@]}"} \ - ${DOCKER_PLATFORM:+--platform ${DOCKER_PLATFORM}} \ - ${CACHE_ARGS[@]+"${CACHE_ARGS[@]}"} \ - ${SCCACHE_ARGS[@]+"${SCCACHE_ARGS[@]}"} \ - ${VERSION_ARGS[@]+"${VERSION_ARGS[@]}"} \ - ${K3S_ARGS[@]+"${K3S_ARGS[@]}"} \ - ${CODEGEN_ARGS[@]+"${CODEGEN_ARGS[@]}"} \ - ${FEATURE_ARGS[@]+"${FEATURE_ARGS[@]}"} \ - --build-arg "CARGO_TARGET_CACHE_SCOPE=${CARGO_TARGET_CACHE_SCOPE}" \ - -f "${DOCKERFILE}" \ - --target "${DOCKER_TARGET}" \ - ${TAG_ARGS[@]+"${TAG_ARGS[@]}"} \ - --provenance=false \ - "$@" \ - ${OUTPUT_ARGS[@]+"${OUTPUT_ARGS[@]}"} \ - . +ce_build \ + ${BUILDER_ARGS[@]+"${BUILDER_ARGS[@]}"} \ + ${DOCKER_PLATFORM:+--platform ${DOCKER_PLATFORM}} \ + ${CACHE_ARGS[@]+"${CACHE_ARGS[@]}"} \ + ${SCCACHE_ARGS[@]+"${SCCACHE_ARGS[@]}"} \ + ${VERSION_ARGS[@]+"${VERSION_ARGS[@]}"} \ + ${K3S_ARGS[@]+"${K3S_ARGS[@]}"} \ + ${CODEGEN_ARGS[@]+"${CODEGEN_ARGS[@]}"} \ + ${FEATURE_ARGS[@]+"${FEATURE_ARGS[@]}"} \ + --build-arg "CARGO_TARGET_CACHE_SCOPE=${CARGO_TARGET_CACHE_SCOPE}" \ + -f "${DOCKERFILE}" \ + --target "${DOCKER_TARGET}" \ + ${TAG_ARGS[@]+"${TAG_ARGS[@]}"} \ + --provenance=false \ + "$@" \ + ${OUTPUT_ARGS[@]+"${OUTPUT_ARGS[@]}"} \ + . diff --git a/tasks/scripts/docker-publish-multiarch.sh b/tasks/scripts/docker-publish-multiarch.sh index 07459c2380..18b7ab889b 100755 --- a/tasks/scripts/docker-publish-multiarch.sh +++ b/tasks/scripts/docker-publish-multiarch.sh @@ -8,6 +8,9 @@ set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/container-engine.sh" + REGISTRY=${DOCKER_REGISTRY:?Set DOCKER_REGISTRY to push multi-arch images (e.g. ghcr.io/myorg)} IMAGE_TAG=${IMAGE_TAG:-dev} PLATFORMS=${DOCKER_PLATFORMS:-linux/amd64,linux/arm64} @@ -22,44 +25,121 @@ if [[ -n "${EXTRA_DOCKER_TAGS_RAW}" ]]; then done fi -BUILDER_NAME=${DOCKER_BUILDER:-multiarch} -if docker buildx inspect "${BUILDER_NAME}" >/dev/null 2>&1; then - echo "Using existing buildx builder: ${BUILDER_NAME}" - docker buildx use "${BUILDER_NAME}" -else - echo "Creating multi-platform buildx builder: ${BUILDER_NAME}..." - docker buildx create --name "${BUILDER_NAME}" --use --bootstrap -fi +# --------------------------------------------------------------------------- +# Docker path: use buildx builders + imagetools for multi-arch +# --------------------------------------------------------------------------- +_publish_multiarch_docker() { + BUILDER_NAME=${DOCKER_BUILDER:-multiarch} + if ce buildx inspect "${BUILDER_NAME}" >/dev/null 2>&1; then + echo "Using existing buildx builder: ${BUILDER_NAME}" + ce buildx use "${BUILDER_NAME}" + else + echo "Creating multi-platform buildx builder: ${BUILDER_NAME}..." + ce buildx create --name "${BUILDER_NAME}" --use --bootstrap + fi -export DOCKER_BUILDER="${BUILDER_NAME}" -export DOCKER_PLATFORM="${PLATFORMS}" -export DOCKER_PUSH=1 -export IMAGE_REGISTRY="${REGISTRY}" + export DOCKER_BUILDER="${BUILDER_NAME}" + export DOCKER_PLATFORM="${PLATFORMS}" + export DOCKER_PUSH=1 + export IMAGE_REGISTRY="${REGISTRY}" -echo "Building multi-arch gateway image..." -tasks/scripts/docker-build-image.sh gateway + echo "Building multi-arch gateway image..." + tasks/scripts/docker-build-image.sh gateway -echo -echo "Building multi-arch cluster image..." -tasks/scripts/docker-build-image.sh cluster + echo + echo "Building multi-arch cluster image..." + tasks/scripts/docker-build-image.sh cluster -TAGS_TO_APPLY=("${EXTRA_TAGS[@]}") -if [[ "${TAG_LATEST}" == "true" ]]; then - TAGS_TO_APPLY+=("latest") -fi + TAGS_TO_APPLY=("${EXTRA_TAGS[@]}") + if [[ "${TAG_LATEST}" == "true" ]]; then + TAGS_TO_APPLY+=("latest") + fi + + if [[ ${#TAGS_TO_APPLY[@]} -gt 0 ]]; then + for component in gateway cluster; do + full_image="${REGISTRY}/${component}" + for tag in "${TAGS_TO_APPLY[@]}"; do + [[ "${tag}" == "${IMAGE_TAG}" ]] && continue + echo "Tagging ${full_image}:${tag}..." + ce_imagetools_create \ + --prefer-index=false \ + -t "${full_image}:${tag}" \ + "${full_image}:${IMAGE_TAG}" + done + done + fi +} + +# --------------------------------------------------------------------------- +# Podman path: build per-platform, assemble manifest lists, push +# --------------------------------------------------------------------------- +_publish_multiarch_podman() { + export IMAGE_REGISTRY="${REGISTRY}" + + # Split comma-separated platforms into an array. + IFS=',' read -ra PLATFORM_LIST <<< "${PLATFORMS}" -if [[ ${#TAGS_TO_APPLY[@]} -gt 0 ]]; then for component in gateway cluster; do - full_image="${REGISTRY}/${component}" + local full_image="${REGISTRY}/${component}" + local manifest_name="${full_image}:${IMAGE_TAG}" + + # Remove any pre-existing manifest list. + ce manifest rm "${manifest_name}" 2>/dev/null || true + ce manifest create "${manifest_name}" + + echo "Building multi-arch ${component} image..." + for platform in "${PLATFORM_LIST[@]}"; do + echo " Building ${component} for ${platform}..." + # Build for each platform and add to the manifest list. + # docker-build-image.sh sources container-engine.sh itself, + # so ce_build is used internally. + DOCKER_PLATFORM="${platform}" \ + DOCKER_PUSH="" \ + IMAGE_TAG="${IMAGE_TAG}" \ + tasks/scripts/docker-build-image.sh "${component}" + + # Tag with a platform-specific suffix for manifest assembly. + local platform_tag="${IMAGE_TAG}-${platform//\//-}" + ce tag "openshell/${component}:${IMAGE_TAG}" "${full_image}:${platform_tag}" + ce push "${full_image}:${platform_tag}" + ce manifest add "${manifest_name}" "${full_image}:${platform_tag}" + done + + echo "Pushing manifest ${manifest_name}..." + ce manifest push --all "${manifest_name}" "docker://${manifest_name}" + + # Apply extra tags. + TAGS_TO_APPLY=("${EXTRA_TAGS[@]}") + if [[ "${TAG_LATEST}" == "true" ]]; then + TAGS_TO_APPLY+=("latest") + fi + for tag in "${TAGS_TO_APPLY[@]}"; do [[ "${tag}" == "${IMAGE_TAG}" ]] && continue echo "Tagging ${full_image}:${tag}..." - docker buildx imagetools create \ - --prefer-index=false \ - -t "${full_image}:${tag}" \ - "${full_image}:${IMAGE_TAG}" + ce manifest create "${full_image}:${tag}" 2>/dev/null || ce manifest rm "${full_image}:${tag}" 2>/dev/null + # Re-create from the primary manifest. + if command -v skopeo >/dev/null 2>&1; then + skopeo copy --all "docker://${manifest_name}" "docker://${full_image}:${tag}" + else + ce manifest create "${full_image}:${tag}" + for platform in "${PLATFORM_LIST[@]}"; do + local platform_tag="${IMAGE_TAG}-${platform//\//-}" + ce manifest add "${full_image}:${tag}" "${full_image}:${platform_tag}" + done + ce manifest push --all "${full_image}:${tag}" "docker://${full_image}:${tag}" + fi done done +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- +if ce_is_docker; then + _publish_multiarch_docker +else + _publish_multiarch_podman fi echo diff --git a/tasks/scripts/sandbox.sh b/tasks/scripts/sandbox.sh index bf7b3cc635..143d9bd90c 100755 --- a/tasks/scripts/sandbox.sh +++ b/tasks/scripts/sandbox.sh @@ -13,6 +13,9 @@ set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/container-engine.sh" + SANDBOX_NAME="dev" CLUSTER_NAME=${CLUSTER_NAME:-$(basename "$PWD")} CONTAINER_NAME="openshell-cluster-${CLUSTER_NAME}" @@ -24,7 +27,7 @@ CMD=(${usage_command:-claude}) # ------------------------------------------------------------------- # 1. Ensure the cluster is running; redeploy if dirty # ------------------------------------------------------------------- -if ! docker ps -q --filter "name=${CONTAINER_NAME}" | grep -q .; then +if ! ce ps -q --filter "name=${CONTAINER_NAME}" | grep -q .; then echo "No running cluster found. Bootstrapping..." mise run cluster else diff --git a/tasks/scripts/vm/build-rootfs-tarball.sh b/tasks/scripts/vm/build-rootfs-tarball.sh index 76e4f62978..57b215aad4 100755 --- a/tasks/scripts/vm/build-rootfs-tarball.sh +++ b/tasks/scripts/vm/build-rootfs-tarball.sh @@ -21,6 +21,8 @@ set -euo pipefail +source "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/container-engine.sh" + ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" ROOTFS_BUILD_DIR="${ROOT}/target/rootfs-build" OUTPUT_DIR="${ROOT}/target/vm-runtime-compressed" @@ -49,17 +51,10 @@ for arg in "$@"; do esac done -# Check for Docker -if ! command -v docker &>/dev/null; then - echo "Error: Docker is required to build the rootfs" >&2 - echo "Please install Docker and try again" >&2 - exit 1 -fi - -# Check if Docker daemon is running -if ! docker info &>/dev/null; then - echo "Error: Docker daemon is not running" >&2 - echo "Please start Docker and try again" >&2 +# Check if container engine is running +if ! ce info &>/dev/null; then + echo "Error: container engine is not running" >&2 + echo "Please start your container engine and try again" >&2 exit 1 fi diff --git a/tasks/scripts/vm/sync-vm-rootfs.sh b/tasks/scripts/vm/sync-vm-rootfs.sh index 727a9dd188..f7b93b971f 100755 --- a/tasks/scripts/vm/sync-vm-rootfs.sh +++ b/tasks/scripts/vm/sync-vm-rootfs.sh @@ -11,6 +11,8 @@ set -euo pipefail +source "$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/container-engine.sh" + ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" SCRIPT_DIR="${ROOT}/crates/openshell-vm/scripts" IMAGE_REPO_BASE="${IMAGE_REPO_BASE:-openshell}" @@ -148,8 +150,8 @@ patch_vm_helmchart "${ROOTFS_DIR}/var/lib/rancher/k3s/server/manifests/openshell # be baked into the rootfs last time it was rebuilt. SERVER_IMAGE_TAR="${ROOTFS_DIR}/var/lib/rancher/k3s/agent/images/openshell-server.tar.zst" SERVER_IMAGE_ID_FILE="${ROOTFS_DIR}/opt/openshell/.gateway-image-id" -if command -v docker >/dev/null 2>&1 && docker image inspect "${SERVER_IMAGE}" >/dev/null 2>&1; then - current_image_id=$(docker image inspect --format '{{.Id}}' "${SERVER_IMAGE}") +if ce image inspect "${SERVER_IMAGE}" >/dev/null 2>&1; then + current_image_id=$(ce image inspect --format '{{.Id}}' "${SERVER_IMAGE}") previous_image_id="" if [ -f "${SERVER_IMAGE_ID_FILE}" ]; then previous_image_id=$(cat "${SERVER_IMAGE_ID_FILE}") @@ -158,7 +160,7 @@ if command -v docker >/dev/null 2>&1 && docker image inspect "${SERVER_IMAGE}" > if [ "${current_image_id}" != "${previous_image_id}" ] || [ ! -f "${SERVER_IMAGE_TAR}" ]; then mkdir -p "$(dirname "${SERVER_IMAGE_TAR}")" "$(dirname "${SERVER_IMAGE_ID_FILE}")" tmp_tar=$(mktemp /tmp/openshell-server-image.XXXXXX) - docker save "${SERVER_IMAGE}" | zstd -f -T0 -3 -o "${tmp_tar}" >/dev/null + ce save "${SERVER_IMAGE}" | zstd -f -T0 -3 -o "${tmp_tar}" >/dev/null mv "${tmp_tar}" "${SERVER_IMAGE_TAR}" printf '%s\n' "${current_image_id}" > "${SERVER_IMAGE_ID_FILE}" echo " updated: /var/lib/rancher/k3s/agent/images/openshell-server.tar.zst" diff --git a/tasks/test.toml b/tasks/test.toml index cf45d2b6bc..177bf5a582 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -48,6 +48,11 @@ depends = ["python:proto", "CLUSTER_GPU=1 cluster"] env = { UV_NO_SYNC = "1", PYTHONPATH = "python" } run = "uv run pytest -o python_files='test_*.py' -m gpu -n ${E2E_PARALLEL:-1} e2e/python" +["e2e:podman"] +description = "Start a Podman-backed gateway and run smoke e2e (requires rootless Podman; pass -- --port=N to override)" +depends = ["build:docker:supervisor-sideload"] +run = "e2e/rust/e2e-podman.sh" + ["e2e:vm"] description = "Start openshell-gateway with the VM compute driver and run the cluster-agnostic smoke e2e" run = "e2e/rust/e2e-vm.sh" From df38d1f66f05dac6a3995844ab96c68c80595265 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Fri, 24 Apr 2026 11:27:02 -0700 Subject: [PATCH 051/142] feat(ci): add Markdown and Mermaid linting (#933) --- .github/workflows/branch-checks.yml | 17 + .gitignore | 3 + .markdownlint-cli2.jsonc | 31 + CLAUDE.md | 2 +- CONTRIBUTING.md | 4 +- SECURITY.md | 2 +- TESTING.md | 2 +- architecture/README.md | 2 +- architecture/custom-vm-runtime.md | 2 +- architecture/gateway-deploy-connect.md | 2 +- architecture/gateway-security.md | 12 +- architecture/gateway-settings.md | 5 +- architecture/gateway-single-node.md | 18 +- architecture/gateway.md | 1 + architecture/inference-routing.md | 2 +- architecture/policy-advisor.md | 1 + architecture/sandbox-connect.md | 2 +- architecture/sandbox-providers.md | 2 +- architecture/sandbox.md | 50 +- architecture/security-policy.md | 9 +- architecture/tui.md | 12 +- crates/openshell-cli/src/doctor_llm_prompt.md | 3 + crates/openshell-driver-vm/README.md | 2 - crates/openshell-vm/README.md | 4 +- crates/openshell-vm/runtime/README.md | 16 +- docs/.markdownlint-cli2.jsonc | 8 + docs/CONTRIBUTING.mdx | 10 +- docs/get-started/tutorials/github-sandbox.mdx | 2 +- .../tutorials/inference-ollama.mdx | 4 +- .../tutorials/local-inference-lmstudio.mdx | 2 + docs/index.mdx | 3 +- docs/inference/configure.mdx | 2 +- docs/reference/gateway-auth.mdx | 2 +- docs/sandboxes/community-sandboxes.mdx | 2 +- examples/gateway-deploy-connect.md | 2 +- examples/local-inference/README.md | 4 +- examples/policy-advisor/README.md | 2 + examples/private-ip-routing/README.md | 2 +- examples/sandbox-policy-quickstart/README.md | 12 +- mise.toml | 1 + rfc/README.md | 2 +- scripts/lint-mermaid/lint-mermaid.mjs | 184 ++ scripts/lint-mermaid/package-lock.json | 1990 +++++++++++++++++ scripts/lint-mermaid/package.json | 11 + scripts/update_license_headers.py | 2 + tasks/ci.toml | 6 +- tasks/markdown.toml | 33 + 47 files changed, 2414 insertions(+), 78 deletions(-) create mode 100644 .markdownlint-cli2.jsonc create mode 100644 docs/.markdownlint-cli2.jsonc create mode 100644 scripts/lint-mermaid/lint-mermaid.mjs create mode 100644 scripts/lint-mermaid/package-lock.json create mode 100644 scripts/lint-mermaid/package.json create mode 100644 tasks/markdown.toml diff --git a/.github/workflows/branch-checks.yml b/.github/workflows/branch-checks.yml index 3d31f6e9f6..0d05c04ea2 100644 --- a/.github/workflows/branch-checks.yml +++ b/.github/workflows/branch-checks.yml @@ -100,3 +100,20 @@ jobs: - name: Test run: mise run test:python + + markdown: + name: Markdown + runs-on: build-amd64 + container: + image: ghcr.io/nvidia/openshell/ci:latest + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + + - name: Install tools + run: mise install + + - name: Lint + run: mise run markdown:lint diff --git a/.gitignore b/.gitignore index d6b4fa3564..08cb97f60d 100644 --- a/.gitignore +++ b/.gitignore @@ -202,3 +202,6 @@ architecture/plans rfc.md .worktrees .z3-trace + +# Markdown/mermaid lint tooling deps +scripts/lint-mermaid/node_modules/ diff --git a/.markdownlint-cli2.jsonc b/.markdownlint-cli2.jsonc new file mode 100644 index 0000000000..47b0a6ae6f --- /dev/null +++ b/.markdownlint-cli2.jsonc @@ -0,0 +1,31 @@ +{ + "globs": [ + "**/*.md", + "**/*.mdx" + ], + "gitignore": true, + "ignores": [ + ".agents/**", + ".claude/**", + ".opencode/**", + ".github/**", + "THIRD-PARTY-NOTICES/**", + "CLAUDE.md" + ], + "config": { + "default": true, + // Allow long lines — prose paragraphs are single-line per project style. + "MD013": false, + // Allow GitHub-rendered HTML commonly used in READMEs (centered logos, + // collapsible sections, keyboard hints). Regular prose HTML still flagged. + "MD033": { "allowed_elements": ["p", "img", "br", "a", "div", "details", "summary", "kbd", "sub", "sup"] }, + // Allow duplicate headings in different sections. + "MD024": { "siblings_only": true }, + // Bare URLs are fine in changelogs and tables. + "MD034": false, + // First line does not need to be a heading. + "MD002": false, + // Repo uses padded table pipes (`| foo | bar |`); rule default is "compact". + "MD060": { "style": "padded" } + } +} diff --git a/CLAUDE.md b/CLAUDE.md index eef4bd20cf..43c994c2d3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1 +1 @@ -@AGENTS.md \ No newline at end of file +@AGENTS.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2852bfa438..c83d32efb6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -258,7 +258,7 @@ See [docs/CONTRIBUTING.mdx](docs/CONTRIBUTING.mdx) for the current docs authorin This project uses [Conventional Commits](https://www.conventionalcommits.org/). All commit messages must follow the format: -``` +```text (): [optional body] @@ -279,7 +279,7 @@ This project uses [Conventional Commits](https://www.conventionalcommits.org/). **Examples:** -``` +```text feat(cli): add --verbose flag to openshell run fix(sandbox): handle timeout errors gracefully docs: update installation instructions diff --git a/SECURITY.md b/SECURITY.md index 728cdb4a1d..9000efe988 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,4 +1,4 @@ -## Security +# Security NVIDIA is dedicated to the security and trust of our software products and services, including all source code repositories managed through our organization. diff --git a/TESTING.md b/TESTING.md index eba31967eb..eab16603ba 100644 --- a/TESTING.md +++ b/TESTING.md @@ -10,7 +10,7 @@ mise run ci # Everything: lint, compile checks, and tests ## Test Layout -``` +```text crates/*/src/ # Inline #[cfg(test)] modules crates/*/tests/ # Rust integration tests python/openshell/ # Python unit tests (*_test.py suffix) diff --git a/architecture/README.md b/architecture/README.md index 570fce660c..36b0a4978f 100644 --- a/architecture/README.md +++ b/architecture/README.md @@ -142,6 +142,7 @@ The connection flow works as follows: 5. The CLI and sandbox exchange SSH traffic bidirectionally through the tunnel. This design provides several benefits: + - Sandbox pods are never directly accessible from outside the cluster. - All access is authenticated and auditable through the gateway. - Session tokens can be revoked to immediately cut off access. @@ -198,7 +199,6 @@ The inference routing system transparently intercepts AI inference API calls fro | Gateway inference service | `crates/openshell-server/src/inference.rs` | Stores cluster inference config, resolves bundles with credentials from provider records | | Proto definitions | `proto/inference.proto` | `ClusterInferenceConfig`, `ResolvedRoute`, bundle RPCs | - ### Container and Build System The platform produces three container images: diff --git a/architecture/custom-vm-runtime.md b/architecture/custom-vm-runtime.md index 4cafe424fb..045ee2e9af 100644 --- a/architecture/custom-vm-runtime.md +++ b/architecture/custom-vm-runtime.md @@ -61,7 +61,7 @@ never binds a host-side TCP listener. `openshell-driver-vm` embeds the VM runtime libraries and the sandbox rootfs as zstd-compressed byte arrays, extracting on demand: -``` +```text ~/.local/share/openshell/vm-runtime// # libkrun / libkrunfw / gvproxy ├── libkrun.{dylib,so} ├── libkrunfw.{5.dylib,so.5} diff --git a/architecture/gateway-deploy-connect.md b/architecture/gateway-deploy-connect.md index ec8153302c..14bb3e90fb 100644 --- a/architecture/gateway-deploy-connect.md +++ b/architecture/gateway-deploy-connect.md @@ -99,7 +99,7 @@ This stores `auth_mode = "plaintext"`, skips mTLS certificate extraction, and by All connection artifacts are stored under `$XDG_CONFIG_HOME/openshell/` (default `~/.config/openshell/`): -``` +```text openshell/ active_gateway # plain text: active gateway name gateways/ diff --git a/architecture/gateway-security.md b/architecture/gateway-security.md index 6baaee88bf..a32c3fb523 100644 --- a/architecture/gateway-security.md +++ b/architecture/gateway-security.md @@ -51,7 +51,7 @@ graph TD The PKI is a single-tier CA hierarchy generated by the `openshell-bootstrap` crate using `rcgen`. All certificates are created in a single pass at cluster bootstrap time. -``` +```text openshell-ca (Self-signed Root CA, O=openshell, CN=openshell-ca) ├── openshell-server (Leaf cert, CN=openshell-server) │ SANs: openshell, openshell.openshell.svc, @@ -94,7 +94,7 @@ The Helm StatefulSet (`deploy/helm/openshell/templates/statefulset.yaml`) mounts Environment variables point the gateway binary to these paths: -``` +```text OPENSHELL_TLS_CERT=/etc/openshell-tls/server/tls.crt OPENSHELL_TLS_KEY=/etc/openshell-tls/server/tls.key OPENSHELL_TLS_CLIENT_CA=/etc/openshell-tls/client-ca/ca.crt @@ -108,7 +108,7 @@ When the gateway creates a sandbox pod (`crates/openshell-server/src/sandbox/mod - A read-only mount at `/etc/openshell-tls/client/` on the agent container. - Environment variables for the sandbox gRPC client: -``` +```text OPENSHELL_TLS_CA=/etc/openshell-tls/client/ca.crt OPENSHELL_TLS_CERT=/etc/openshell-tls/client/tls.crt OPENSHELL_TLS_KEY=/etc/openshell-tls/client/tls.key @@ -119,7 +119,7 @@ OPENSHELL_ENDPOINT=https://openshell.openshell.svc.cluster.local:8080 The CLI's copy of the client certificate bundle is written to: -``` +```text $XDG_CONFIG_HOME/openshell/gateways//mtls/ ├── ca.crt ├── tls.crt @@ -183,7 +183,7 @@ The gateway supports three transport modes: ### Connection Flow -``` +```text TCP accept → TLS handshake (mandatory client cert in mTLS mode, optional in dual-auth mode) → hyper auto-negotiates HTTP/1.1 or HTTP/2 via ALPN @@ -225,10 +225,12 @@ Sandbox pods connect back to the gateway at startup to fetch their policy and pr | `OPENSHELL_TLS_KEY` | `/etc/openshell-tls/client/tls.key` | These are used to build a `tonic::transport::ClientTlsConfig` with: + - `ca_certificate()` -- verifies the server's certificate against the cluster CA. - `identity()` -- presents the shared client certificate for mTLS. The sandbox calls two RPCs over this authenticated channel: + - `GetSandboxSettings` -- fetches the YAML policy that governs the sandbox's behavior. - `GetSandboxProviderEnvironment` -- fetches provider credentials as environment variables. diff --git a/architecture/gateway-settings.md b/architecture/gateway-settings.md index ef9538f5b0..4ae191de08 100644 --- a/architecture/gateway-settings.md +++ b/architecture/gateway-settings.md @@ -45,6 +45,7 @@ pub const REGISTERED_SETTINGS: &[RegisteredSetting] = &[ The reserved key `policy` is excluded from the registry. It is handled by dedicated policy commands and stored as a hex-encoded protobuf `SandboxPolicy` in the global settings' `Bytes` variant. Attempts to set or delete the `policy` key through settings commands are rejected. Helper functions: + - `setting_for_key(key)` -- look up a `RegisteredSetting` by name, returns `None` for unknown keys - `registered_keys_csv()` -- comma-separated list of valid keys for error messages - `parse_bool_like(raw)` -- flexible bool parsing from CLI string input @@ -83,6 +84,7 @@ The `UpdateSettings` RPC multiplexes policy and setting mutations through a sing | `global` | `bool` | Target gateway-global scope instead of sandbox scope | Validation rules: + - `policy` and `setting_key` cannot both be present - At least one of `policy` or `setting_key` must be present - `delete_setting` cannot be combined with a `policy` payload @@ -266,7 +268,7 @@ This prevents conflicting values at different scopes. An operator must delete a When a global policy is set, sandbox-scoped policy updates via `UpdateSettings` are rejected with `FailedPrecondition`: -``` +```text policy is managed globally; delete global policy before sandbox policy update ``` @@ -442,6 +444,7 @@ openshell policy get --global --full All `--global` mutations require human-in-the-loop confirmation via an interactive prompt. The `--yes` flag bypasses the prompt for scripted/CI usage. In non-interactive mode (no TTY), `--yes` is required -- otherwise the command fails with an error. The confirmation message varies: + - **Global setting set**: warns that this will override sandbox-level values for the key - **Global setting delete**: warns that this re-enables sandbox-level management - **Global policy set**: warns that this overrides all sandbox policies diff --git a/architecture/gateway-single-node.md b/architecture/gateway-single-node.md index 6389c728e1..01b69b2f5d 100644 --- a/architecture/gateway-single-node.md +++ b/architecture/gateway-single-node.md @@ -168,7 +168,7 @@ For the target daemon (local or remote): - k3s server command: `server --disable=traefik --tls-san=127.0.0.1 --tls-san=localhost --tls-san=host.docker.internal` plus computed extra SANs. - Privileged mode. - Volume bind mount: `openshell-cluster-{name}:/var/lib/rancher/k3s`. - - Network: `openshell-cluster-{name}` (per-gateway bridge network). + - Network: `openshell-cluster-{name}` (per-gateway bridge network). - Extra host: `host.docker.internal:host-gateway`. - The cluster entrypoint prefers the resolved IPv4 for `host.docker.internal` when populating sandbox pod `hostAliases`, then falls back to the container default gateway. This keeps sandbox host aliases working on Docker Desktop, where the host-reachable IP differs from the bridge gateway. - Port mappings: @@ -230,7 +230,7 @@ After deploy, the CLI calls `save_active_gateway(name)`, writing the gateway nam The cluster image is defined by target `cluster` in `deploy/docker/Dockerfile.images`: -``` +```text Base: rancher/k3s:v1.35.2-k3s1 ``` @@ -242,6 +242,7 @@ Layers added: 4. Kubernetes manifests: `deploy/kube/manifests/*.yaml` -> `/opt/openshell/manifests/` Bundled manifests include: + - `openshell-helmchart.yaml` (OpenShell Helm chart auto-deploy) - `envoy-gateway-helmchart.yaml` (Envoy Gateway for Gateway API) - `agent-sandbox.yaml` @@ -363,9 +364,10 @@ flowchart LR 4. Force-remove the per-gateway network via `force_remove_network()`, disconnecting any stale endpoints first. **CLI layer** (`gateway_destroy()` in `run.rs` additionally): - -6. Remove the metadata JSON file via `remove_gateway_metadata()`. -7. Clear the active gateway reference if it matches the destroyed gateway. + +5. Remove the metadata JSON file via `remove_gateway_metadata()`. +6. Clear the active gateway reference if it matches the destroyed gateway. + ## Idempotency and Error Behavior @@ -378,8 +380,8 @@ flowchart LR - Docker API failures from inspect/create/start/remove. - SSH connection failures when creating the remote Docker client. - Health check timeout (6 min) with recent container logs. - - Container exit during any polling phase (health, mTLS) with diagnostic information (exit code, OOM status, recent logs). - - mTLS secret polling timeout (3 min). + - Container exit during any polling phase (health, mTLS) with diagnostic information (exit code, OOM status, recent logs). + - mTLS secret polling timeout (3 min). - Local image ref without registry prefix: clear error with build instructions rather than a failed Docker Hub pull. ## Auto-Bootstrap from `sandbox create` @@ -436,7 +438,7 @@ Environment variables that affect bootstrap behavior when set on the host: Artifacts stored under `$XDG_CONFIG_HOME/openshell/` (default `~/.config/openshell/`): -``` +```text openshell/ active_gateway # plain text: active gateway name gateways/ diff --git a/architecture/gateway.md b/architecture/gateway.md index 9e9da67859..5fb82717aa 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -510,6 +510,7 @@ graph LR ``` All buses use `tokio::sync::broadcast` channels keyed by sandbox ID. Buffer sizes: + - `SandboxWatchBus`: 128 (signals only, no payload -- just `()`) - `TracingLogBus`: 1024 (full `SandboxStreamEvent` payloads) - `PlatformEventBus`: 1024 (full `SandboxStreamEvent` payloads) diff --git a/architecture/inference-routing.md b/architecture/inference-routing.md index c0c42f4f6a..4d7b0f5171 100644 --- a/architecture/inference-routing.md +++ b/architecture/inference-routing.md @@ -214,7 +214,7 @@ This eliminates full-body buffering for streaming responses (SSE). Time-to-first When the proxy truncates a streaming response, it injects an SSE error event via `format_sse_error()` (in `crates/openshell-sandbox/src/l7/inference.rs`) before sending the HTTP chunked terminator: -``` +```text data: {"error":{"message":"","type":"proxy_stream_error"}} ``` diff --git a/architecture/policy-advisor.md b/architecture/policy-advisor.md index 6d2728332b..c70bfcbd37 100644 --- a/architecture/policy-advisor.md +++ b/architecture/policy-advisor.md @@ -208,6 +208,7 @@ The TUI sandbox screen includes a "Network Rules" panel accessible via `[r]` fro - Expanded detail popup with full binary path, rationale, security notes, and proposed rule Keybindings are state-aware: + - **Pending** → `[a]` approve, `[x]` reject, `[A]` approve all - **Approved** → `[x]` revoke - **Rejected** → `[a]` approve diff --git a/architecture/sandbox-connect.md b/architecture/sandbox-connect.md index f9a0c491e0..2f5e62b2b5 100644 --- a/architecture/sandbox-connect.md +++ b/architecture/sandbox-connect.md @@ -182,7 +182,7 @@ sequenceDiagram CLI->>GW: CreateSshSession(sandbox_id) GW-->>CLI: token, gateway_host, gateway_port, scheme, connect_path - Note over CLI: Builds ProxyCommand string; exec()s ssh + Note over CLI: Builds ProxyCommand string: exec()s ssh User->>CLI: ssh spawns ssh-proxy subprocess CLI->>GW: CONNECT /connect/ssh
X-Sandbox-Id, X-Sandbox-Token diff --git a/architecture/sandbox-providers.md b/architecture/sandbox-providers.md index fe5d48a97d..088bd75920 100644 --- a/architecture/sandbox-providers.md +++ b/architecture/sandbox-providers.md @@ -333,7 +333,7 @@ the full implementation details, encoding rules, and security properties. ### End-to-End Flow -``` +```text CLI: openshell sandbox create -- claude | +-- detect_provider_from_command(["claude"]) -> "claude" diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 5104a6dccd..2c0b5fe1ba 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -116,18 +116,18 @@ flowchart TD 8. **SSH server** (optional): If `--ssh-socket-path` is provided, spawn an async task running `ssh::run_ssh_server()` with the policy, workdir, netns FD, proxy URL, CA paths, and provider env. The value is a filesystem path to the Unix socket the embedded sshd binds. The supervisor waits on a readiness `oneshot` channel before proceeding so that exec requests arriving immediately after pod-ready cannot race against socket bind. -8a. **Supervisor session** (gRPC mode + SSH socket only): If `--sandbox-id`, `--openshell-endpoint`, and an SSH socket path are all set, spawn `supervisor_session::spawn()`. This task opens a persistent outbound bidirectional gRPC stream to the gateway and bridges inbound relay requests to the local SSH daemon. See [Supervisor Session](#supervisor-session) for the full protocol. +9. **Supervisor session** (gRPC mode + SSH socket only): If `--sandbox-id`, `--openshell-endpoint`, and an SSH socket path are all set, spawn `supervisor_session::spawn()`. This task opens a persistent outbound bidirectional gRPC stream to the gateway and bridges inbound relay requests to the local SSH daemon. See [Supervisor Session](#supervisor-session) for the full protocol. -9. **Child process spawning** (`ProcessHandle::spawn()`): - - Build `tokio::process::Command` with inherited stdio and `kill_on_drop(true)` - - Set environment variables: `OPENSHELL_SANDBOX=1`, provider credentials, proxy URLs, TLS trust store paths - - Pre-exec closure (async-signal-safe): `setpgid` (if non-interactive) -> `setns` (enter netns) -> `drop_privileges` -> `sandbox::apply` (Landlock + seccomp) +10. **Child process spawning** (`ProcessHandle::spawn()`): + - Build `tokio::process::Command` with inherited stdio and `kill_on_drop(true)` + - Set environment variables: `OPENSHELL_SANDBOX=1`, provider credentials, proxy URLs, TLS trust store paths + - Pre-exec closure (async-signal-safe): `setpgid` (if non-interactive) -> `setns` (enter netns) -> `drop_privileges` -> `sandbox::apply` (Landlock + seccomp) -10. **Store entrypoint PID**: `entrypoint_pid.store(pid, Ordering::Release)` so the proxy can resolve TCP peer identity via `/proc`. +11. **Store entrypoint PID**: `entrypoint_pid.store(pid, Ordering::Release)` so the proxy can resolve TCP peer identity via `/proc`. -11. **Spawn policy poll task** (gRPC mode only): If `sandbox_id`, `openshell_endpoint`, and an OPA engine are all present, spawn `run_policy_poll_loop()` as a background tokio task. This task polls the gateway for policy updates and hot-reloads the OPA engine when a new version is detected. See [Policy Reload Lifecycle](#policy-reload-lifecycle) for details. +12. **Spawn policy poll task** (gRPC mode only): If `sandbox_id`, `openshell_endpoint`, and an OPA engine are all present, spawn `run_policy_poll_loop()` as a background tokio task. This task polls the gateway for policy updates and hot-reloads the OPA engine when a new version is detected. See [Policy Reload Lifecycle](#policy-reload-lifecycle) for details. -12. **Wait with timeout**: If `--timeout > 0`, wrap `handle.wait()` in `tokio::time::timeout()`. On timeout, kill the process and return exit code 124. +13. **Wait with timeout**: If `--timeout > 0`, wrap `handle.wait()` in `tokio::time::timeout()`. On timeout, kill the process and return exit code 124. ## Policy Model @@ -244,6 +244,7 @@ Two evaluation methods exist: `evaluate_network()` for the legacy bool-based pat #### `evaluate_network(input: &NetworkInput) -> Result` Input JSON shape: + ```json { "exec": { @@ -259,6 +260,7 @@ Input JSON shape: ``` Evaluates three Rego rules: + 1. `data.openshell.sandbox.allow_network` -> bool 2. `data.openshell.sandbox.deny_reason` -> string 3. `data.openshell.sandbox.matched_network_policy` -> string (or `Undefined`) @@ -273,6 +275,7 @@ Uses the same input JSON shape as `evaluate_network()`. Evaluates the `data.open - `"deny"` -- network connections not allowed by policy The Rego logic: + 1. If `network_policy_for_request` exists (endpoint + binary match), return `"allow"` 2. Default: `"deny"` @@ -394,6 +397,7 @@ pub struct SettingsPollResult { ``` Methods: + - **`connect(endpoint)`**: Establish an mTLS channel and return a new client. - **`poll_settings(sandbox_id)`**: Call `GetSandboxSettings` RPC and return a `SettingsPollResult` containing policy payload (optional), policy metadata, effective config revision, policy source, global policy version, and the effective settings map (for diff logging). - **`report_policy_status(sandbox_id, version, loaded, error_msg)`**: Call `ReportPolicyStatus` RPC with the appropriate `PolicyStatus` enum value (`Loaded` or `Failed`). @@ -404,6 +408,7 @@ Methods: The gateway assigns a monotonically increasing version number to each sandbox policy revision. `GetSandboxSettingsResponse` carries the full effective configuration: policy payload, effective settings map (with per-key scope indicators), a `config_revision` fingerprint that changes when any effective input changes (policy, settings, or source), and a `policy_source` field indicating whether the policy came from the sandbox's own history or from a global override. Proto messages involved: + - `GetSandboxSettingsResponse` (`proto/sandbox.proto`): `policy`, `version`, `policy_hash`, `settings` (map of `EffectiveSetting`), `config_revision`, `policy_source`, `global_policy_version` - `EffectiveSetting` (`proto/sandbox.proto`): `SettingValue value`, `SettingScope scope` - `SettingScope` enum: `UNSPECIFIED`, `SANDBOX`, `GLOBAL` @@ -449,6 +454,7 @@ Landlock restricts the child process's filesystem access to an explicit allowlis 7. Call `ruleset.restrict_self()` -- this applies to the calling process and all descendants Kernel-level error behavior (e.g., Landlock ABI unavailable) depends on `LandlockCompatibility`: + - `BestEffort`: Log a warning and continue without filesystem isolation - `HardRequirement`: Return a fatal error, aborting the sandbox @@ -463,6 +469,7 @@ Seccomp provides three layers of syscall restriction: socket domain blocks, unco **Skipped entirely** in `Allow` mode. Setup: + 1. `prctl(PR_SET_NO_NEW_PRIVS, 1)` -- required before seccomp 2. `seccompiler::apply_filter()` with default action `Allow` and per-rule action `Errno(EPERM)` @@ -512,7 +519,7 @@ The network namespace creates an isolated network stack where the sandboxed proc #### Topology -``` +```text HOST NAMESPACE SANDBOX NAMESPACE ----------------- ----------------- veth-h-{uuid} veth-s-{uuid} @@ -540,6 +547,7 @@ Each step has rollback on failure -- if any `ip` command fails, previously creat #### Cleanup on drop `NetworkNamespace` implements `Drop`: + 1. Close the namespace FD 2. Delete the host-side veth (`ip link delete veth-h-{id}`) -- this automatically removes the peer 3. Delete the namespace (`ip netns delete sandbox-{id}`) @@ -903,6 +911,7 @@ The sandbox is designed to operate both as part of a cluster and as a standalone #### Cluster mode graceful degradation In cluster mode, `fetch_inference_bundle()` failures are handled based on the error type: + - gRPC `PermissionDenied` or `NotFound` (detected via error message string matching): sandbox has no inference policy -- inference routing is silently disabled. - Other errors: logged as a warning, inference routing is disabled. - Empty initial route bundle: inference routing stays enabled with an empty cache and background refresh continues. @@ -1029,12 +1038,14 @@ Expansion happens in `expand_access_presets()` before the Rego engine loads the `validate_l7_policies()` runs at engine load time and returns `(errors, warnings)`: **Errors** (block startup): + - `rules` and `access` both specified on same endpoint - `protocol` specified without `rules` or `access` - `protocol: sql` with `enforcement: enforce` (SQL parsing not available in v1) - Empty `rules` array (would deny all traffic) **Warnings** (logged): + - `tls: terminate` or `tls: passthrough` on any endpoint (deprecated — TLS termination is now automatic; use `tls: skip` to disable) - `tls: skip` with L7 rules on port 443 (L7 inspection cannot work on encrypted traffic) - Unknown HTTP method in rules @@ -1046,23 +1057,27 @@ Expansion happens in `expand_access_presets()` before the Rego engine loads the TLS termination is automatic. The proxy peeks the first bytes of every CONNECT tunnel and terminates TLS whenever a ClientHello is detected. This enables credential injection and L7 inspection on all HTTPS endpoints without requiring explicit `tls: terminate` in the policy. The `tls` field defaults to `Auto`; use `tls: skip` to opt out entirely (e.g., for client-cert mTLS to upstream). **Ephemeral CA lifecycle:** + 1. At sandbox startup, `SandboxCa::generate()` creates a self-signed CA (CN: "OpenShell Sandbox CA") using `rcgen` 2. The CA cert PEM and a combined bundle (system CAs + sandbox CA) are written to `/etc/openshell-tls/` 3. The sandbox CA cert path is set as `NODE_EXTRA_CA_CERTS` (additive for Node.js) 4. The combined bundle is set as `SSL_CERT_FILE`, `REQUESTS_CA_BUNDLE`, `CURL_CA_BUNDLE` (replaces defaults for OpenSSL, Python requests, curl) **TLS auto-detection** (`looks_like_tls()`): + - Peeks up to 8 bytes from the client stream - Checks for TLS ClientHello pattern: byte 0 = `0x16` (ContentType::Handshake), byte 1 = `0x03` (TLS major version), byte 2 ≤ `0x04` (minor version, covering SSL 3.0 through TLS 1.3) - Returns `false` for plaintext HTTP, SSH, or other binary protocols **Per-hostname leaf cert generation:** + - `CertCache` maps hostnames to `CertifiedLeaf` structs (cert chain + private key) - First request for a hostname generates a leaf cert signed by the sandbox CA via `rcgen` - Cache has a hard limit of 256 entries; on overflow, the entire cache is cleared (sufficient for sandbox scale) - Each leaf cert chain contains two certs: the leaf and the CA **Connection flow (when TLS is detected):** + 1. `tls_terminate_client()`: Accept TLS from the sandboxed client using a `ServerConfig` with the hostname-specific leaf cert. ALPN: `http/1.1`. 2. `tls_connect_upstream()`: Connect TLS to the real upstream using a `ClientConfig` with Mozilla root CAs (`webpki_roots`) and system CA certificates. ALPN: `http/1.1`. 3. Proxy now holds plaintext on both sides. If L7 config is present, runs `relay_with_inspection()`. Otherwise, runs `relay_passthrough_with_credentials()` for credential injection without L7 evaluation. @@ -1233,11 +1248,13 @@ each cached entry stores: - File fingerprint (`len`, `mtime`, `ctime`, and on Unix `dev` + `inode`) `verify_or_cache(path)`: + - **First call for a path**: Compute SHA256 via `procfs::file_sha256()`, store as the "golden" hash plus fingerprint, return the hash. - **Subsequent calls, unchanged fingerprint**: Return cached hash without re-hashing the file. - **Subsequent calls, changed fingerprint**: Recompute SHA256 and compare with cached value. Return `Ok(hash)` on match; return `Err` on mismatch (binary tampered/replaced mid-sandbox). The TOFU model means: + - No hashes are specified in policy data -- the first observed binary is trusted - Once trusted, the binary cannot change for the sandbox's lifetime - Both the immediate binary and all ancestor binaries are TOFU-verified @@ -1279,12 +1296,14 @@ Both IPv4 (`/proc/{pid}/net/tcp`) and IPv6 (`/proc/{pid}/net/tcp6`) tables are c Wraps `tokio::process::Child` + PID. Platform-specific `spawn()` methods delegate to `spawn_impl()`. **Environment setup** (both Linux and non-Linux): + - `OPENSHELL_SANDBOX=1` (always set) - Provider credentials (from `GetSandboxProviderEnvironment` RPC) - Proxy URLs: `HTTP_PROXY`, `HTTPS_PROXY`, `ALL_PROXY` (uppercase for curl/wget), `NO_PROXY=127.0.0.1,localhost,::1` for localhost bypass, `http_proxy`, `https_proxy`, `grpc_proxy` (lowercase for gRPC C-core), `no_proxy=127.0.0.1,localhost,::1`, `NODE_USE_ENV_PROXY=1` (required for Node.js built-in `fetch`/`http` clients to honor proxy env vars) - TLS trust store: `NODE_EXTRA_CA_CERTS` (standalone CA cert), `SSL_CERT_FILE`, `REQUESTS_CA_BUNDLE`, `CURL_CA_BUNDLE` (combined bundle) **Pre-exec closure** (runs in child after fork, before exec -- async-signal-safe): + 1. `setpgid(0, 0)` if non-interactive (create new process group) 2. `setns(fd, CLONE_NEWNET)` to enter network namespace (Linux only) 3. `drop_privileges(policy)`: `initgroups()` -> `setgid()` -> `setuid()` @@ -1295,6 +1314,7 @@ Wraps `tokio::process::Child` + PID. Platform-specific `spawn()` methods delegat ### `drop_privileges()` Resolves user/group names from policy, then: + 1. `initgroups()` to set supplementary groups (Linux only, not macOS) 2. `setgid()` to target group 3. Verify `getegid()` matches the target GID @@ -1352,6 +1372,7 @@ The `SshHandler` implements `russh::server::Handler`: ### PTY child process `spawn_pty_shell()`: + 1. `openpty()` to create a master/slave PTY pair 2. Build `std::process::Command` (not tokio) with slave FDs for stdin/stdout/stderr 3. Set environment: `OPENSHELL_SANDBOX=1`, `HOME=/sandbox`, `USER=sandbox`, `TERM={negotiated}`, proxy URLs, TLS trust store paths, provider credentials @@ -1570,10 +1591,12 @@ The sandbox uses `miette` for error reporting and `thiserror` for typed errors. ## Logging Dual-output logging is configured in `main.rs`: + - **stdout**: Filtered by `--log-level` (default `warn`), uses ANSI colors - **`/var/log/openshell.log`**: Fixed at `info` level, no ANSI, non-blocking writer Key structured log events: + - `CONNECT`: One per proxy CONNECT request (for non-`inference.local` targets) with full identity context. Inference interception failures produce a separate `info!()` log with `action=deny` and the denial reason. - `BYPASS_DETECT`: One per detected direct connection attempt that bypassed the HTTP CONNECT proxy. Includes destination, protocol, process identity (best-effort), and remediation hint. Emitted at `warn` level. - `L7_REQUEST`: One per L7-inspected request with method, path, and decision @@ -1608,6 +1631,7 @@ flowchart LR ``` Two log sources feed the same `TracingLogBus`: + - **Gateway logs** (`source: "gateway"`): Generated by the server's `SandboxLogLayer` tracing layer when server-side code emits events containing a `sandbox_id` field. These capture reconciliation, provisioning, and management operations. - **Sandbox logs** (`source: "sandbox"`): Pushed from the sandbox supervisor via the `PushSandboxLogs` client-streaming RPC. These capture proxy decisions, policy reloads, process lifecycle, and all other sandbox-internal tracing events. @@ -1626,6 +1650,7 @@ pub struct LogPushLayer { ``` Key behaviors: + - **Level filtering**: Defaults to `INFO`. Configurable via the `OPENSHELL_LOG_PUSH_LEVEL` environment variable (accepts `trace`, `debug`, `info`, `warn`, `error`). Events above the configured level are silently discarded. - **Best-effort delivery**: Uses `try_send()` on the mpsc channel. If the channel is full (1024 lines buffered), the event is dropped. Logging never blocks the sandbox supervisor. - **Structured fields**: Implements a `LogVisitor` that collects all tracing key-value fields (e.g., `dst_host`, `action`, `policy`) into a `HashMap`. The `message` field is extracted separately; all other fields go into `SandboxLogLine.fields`. @@ -1662,6 +1687,7 @@ The background task batches log lines and streams them to the gateway: **File:** `crates/openshell-server/src/grpc.rs` (`push_sandbox_logs`) The `PushSandboxLogs` RPC handler processes each batch: + 1. Validates `sandbox_id` is non-empty (skips empty batches). 2. Iterates over `batch.logs`, capped at 100 lines per batch to prevent abuse. 3. Forces `log.source = "sandbox"` on every line -- the sandbox cannot claim to be the gateway. @@ -1673,6 +1699,7 @@ The `PushSandboxLogs` RPC handler processes each batch: **File:** `crates/openshell-server/src/tracing_bus.rs` `publish_external()` wraps the `SandboxLogLine` in a `SandboxStreamEvent` and calls the internal `publish()` method, which: + 1. Sends the event to the per-sandbox `broadcast::Sender` (capacity 1024). Subscribers (active `WatchSandbox` streams) receive the event immediately. 2. Appends the event to the per-sandbox tail buffer (`VecDeque`), capped at 2000 lines. Overflow evicts the oldest entry. @@ -1746,12 +1773,13 @@ Filtering is implemented server-side. For `WatchSandbox`, filters apply to both `print_log_line()` in `crates/openshell-cli/src/run.rs` formats each log line: -``` +```text [timestamp] [source ] [level] [target] message key=value key=value ``` Example output: -``` + +```text [1708891234.567] [sandbox] [INFO ] [openshell_sandbox::proxy] CONNECT api.example.com:443 dst_host=api.example.com action=allow [1708891234.890] [gateway] [INFO ] [openshell_server::grpc] ReportPolicyStatus: sandbox reported policy load result ``` diff --git a/architecture/security-policy.md b/architecture/security-policy.md index e3ec6abbc9..8afef7ae3e 100644 --- a/architecture/security-policy.md +++ b/architecture/security-policy.md @@ -283,6 +283,7 @@ The `--global` flag on `policy set`, `policy delete`, `policy list`, and `policy Both `set` and `delete` require interactive confirmation (or `--yes` to bypass). The `--wait` flag is rejected for global policy updates: `"--wait is not supported for global policies; global policies are effective immediately"`. When a global policy is active, sandbox-scoped policy mutations are blocked: + - `policy set ` returns `FailedPrecondition: "policy is managed globally"` - `policy update ` returns `FailedPrecondition: "policy is managed globally"` - `rule approve`, `rule approve-all` return `FailedPrecondition: "cannot approve rules while a global policy is active"` @@ -764,7 +765,7 @@ If any condition fails, the proxy returns `403 Forbidden`. **Logging**: Forward proxy requests are logged distinctly from CONNECT: -``` +```text FORWARD method=GET dst_host=10.86.8.223 dst_port=8000 path=/screenshot/ action=allow policy=computer-control ``` @@ -823,8 +824,8 @@ TLS termination is automatic. The proxy peeks the first bytes of every CONNECT t | ------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `tls` absent or `""` (default) | **Auto-detect**: The proxy peeks the first bytes of the tunnel. If TLS is detected (ClientHello pattern), the proxy terminates TLS transparently (MITM), enabling credential injection and L7 inspection. If plaintext HTTP is detected, the proxy inspects directly. If neither, traffic is relayed raw. | | `tls: "skip"` | **Explicit opt-out**: No TLS detection, no termination, no credential injection. The tunnel is a raw `copy_bidirectional` relay. Use for client-cert mTLS to upstream or non-standard binary protocols. | -| `tls: "terminate"` *(deprecated)* | Treated as auto-detect. Emits a deprecation warning: "TLS termination is now automatic. Use `tls: skip` to explicitly disable." | -| `tls: "passthrough"` *(deprecated)* | Treated as auto-detect. Emits the same deprecation warning. | +| `tls: "terminate"` _(deprecated)_ | Treated as auto-detect. Emits a deprecation warning: "TLS termination is now automatic. Use `tls: skip` to explicitly disable." | +| `tls: "passthrough"` _(deprecated)_ | Treated as auto-detect. Emits the same deprecation warning. | **Prerequisites for TLS termination (auto-detect path)**: @@ -837,6 +838,7 @@ TLS termination is automatic. The proxy peeks the first bytes of every CONNECT t **Credential injection**: When TLS is auto-terminated but no L7 policy is configured (no `protocol` field), the proxy enters a passthrough relay that rewrites credential placeholders in HTTP headers (via `SecretResolver`) and logs requests for observability, but does not evaluate L7 OPA rules. This means credential injection works on all HTTPS endpoints automatically. **Validation warnings**: + - `tls: terminate` or `tls: passthrough`: deprecated, emits a warning. - `tls: skip` with `protocol: rest` on port 443: emits a warning ("L7 inspection cannot work on encrypted traffic"). @@ -1194,6 +1196,7 @@ With a matching sandbox `/etc/hosts` entry such as `192.168.1.105 searxng.local` #### `allowed_ips` Format Entries can be: + - **CIDR notation**: `10.0.5.0/24`, `172.16.0.0/12`, `192.168.1.0/24` - **Exact IP**: `10.0.5.20` (treated as `/32` for IPv4 or `/128` for IPv6) diff --git a/architecture/tui.md b/architecture/tui.md index 1a83e96d1a..00e53a8296 100644 --- a/architecture/tui.md +++ b/architecture/tui.md @@ -25,7 +25,7 @@ No separate configuration files or authentication are needed. The TUI divides the terminal into four horizontal regions: -``` +```text ┌─────────────────────────────────────────────────────────────────┐ │ OpenShell ─ my-cluster ─ Dashboard ● Healthy │ ← title bar ├─────────────────────────────────────────────────────────────────┤ @@ -58,10 +58,11 @@ The dashboard is divided into a top info pane and a middle pane with two tabs: - **Global Settings** — gateway-global runtime settings (fetched via `GetGatewaySettings`). **Health status** indicators: - - `●` **Healthy** (green) — everything is running normally. - - `◐` **Degraded** (yellow) — the cluster is up but something needs attention. - - `○` **Unhealthy** (red) — the cluster is not operating correctly. - - `…` — still connecting or status unknown. + +- `●` **Healthy** (green) — everything is running normally. +- `◐` **Degraded** (yellow) — the cluster is up but something needs attention. +- `○` **Unhealthy** (red) — the cluster is not operating correctly. +- `…` — still connecting or status unknown. **Global policy indicator**: When a global policy is active, the gateway row shows `Global Policy Active (vN)` in yellow (the `status_warn` style). The TUI detects this by polling `ListSandboxPolicies` with `global: true, limit: 1` on each tick and checking if the latest revision has `PolicyStatus::Loaded`. See `crates/openshell-tui/src/ui/dashboard.rs`. @@ -174,6 +175,7 @@ The TUI supports creating sandboxes with port forwarding directly from the creat Forwarded ports are displayed in the **NOTES** column of the sandbox table as `fwd:8080,3000` and in the **Forwards** row of the sandbox detail view. Port forwarding lifecycle: + - **On create**: The TUI polls for sandbox readiness (up to 30 attempts at 2-second intervals), then spawns SSH tunnels. - **On delete**: Any active forwards for the sandbox are automatically stopped before deletion. - **PID tracking**: Forward PIDs are stored in `~/.config/openshell/forwards/-.pid`, shared with the CLI. diff --git a/crates/openshell-cli/src/doctor_llm_prompt.md b/crates/openshell-cli/src/doctor_llm_prompt.md index 4d4a6b64c7..a30d8981f9 100644 --- a/crates/openshell-cli/src/doctor_llm_prompt.md +++ b/crates/openshell-cli/src/doctor_llm_prompt.md @@ -68,11 +68,13 @@ Before running commands, establish: ### Step 0: Quick Connectivity Check Run `openshell status` first. This immediately reveals: + - Which gateway and endpoint the CLI is targeting - Whether the CLI can reach the server (mTLS handshake success/failure) - The server version if connected Common errors at this stage: + - **`tls handshake eof`**: The server isn't running or mTLS credentials are missing/mismatched - **`connection refused`**: The container isn't running or port mapping is broken - **`No gateway configured`**: No gateway has been deployed yet @@ -222,6 +224,7 @@ openshell doctor exec -- kubectl -n openshell get secret openshell-client-tls -o ``` Common mTLS issues: + - **Secrets missing**: The `openshell` namespace may not have been created yet (Helm controller race). Bootstrap waits up to 2 minutes for the namespace. - **mTLS mismatch after manual secret deletion**: Delete all three secrets and redeploy — bootstrap will regenerate and restart the workload. - **CLI can't connect after redeploy**: Check that `~/.config/openshell/gateways//mtls/` contains `ca.crt`, `tls.crt`, `tls.key` and that they were updated at deploy time. diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index 0a82999d33..44f326c3b3 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -31,12 +31,10 @@ Sandbox guests execute `/opt/openshell/bin/openshell-sandbox` as PID 1 inside th ## Quick start (recommended) - ```shell mise run gateway:vm ``` - First run takes a few minutes while `mise run vm:setup` stages libkrun/libkrunfw/gvproxy and `mise run vm:rootfs -- --base` builds the embedded rootfs. Subsequent runs are cached. To keep the Unix socket path under macOS `SUN_LEN`, `mise run gateway:vm` and `start.sh` default the state dir to `/tmp/openshell-vm-driver-dev-$USER-port-$PORT/` (SQLite DB + per-sandbox rootfs + `compute-driver.sock`) unless `OPENSHELL_VM_DRIVER_STATE_DIR` is set. The wrapper also prints the recommended gateway name (`vm-driver-port-$PORT` by default) plus the exact repo-local `scripts/bin/openshell gateway add` and `scripts/bin/openshell gateway select` commands to use from another terminal. This avoids accidentally hitting an older `openshell` binary elsewhere on your `PATH`. It also exports `OPENSHELL_DRIVER_DIR=$PWD/target/debug` before starting the gateway so local dev runs use the freshly built `openshell-driver-vm` instead of an older installed copy from `~/.local/libexec/openshell` or `/usr/local/libexec`. diff --git a/crates/openshell-vm/README.md b/crates/openshell-vm/README.md index b23cc3c271..32632e3492 100644 --- a/crates/openshell-vm/README.md +++ b/crates/openshell-vm/README.md @@ -121,7 +121,7 @@ Each instance gets its own extracted rootfs under `~/.local/share/openshell/open ## CLI Reference -``` +```text openshell-vm [OPTIONS] [COMMAND] Options: @@ -179,7 +179,7 @@ FROM_SOURCE=1 mise run vm:setup ## Architecture -``` +```text Host (macOS / Linux) openshell-vm binary |-- Embedded runtime (libkrun, libkrunfw, gvproxy, rootfs.tar.zst) diff --git a/crates/openshell-vm/runtime/README.md b/crates/openshell-vm/runtime/README.md index f439811028..76646a5baa 100644 --- a/crates/openshell-vm/runtime/README.md +++ b/crates/openshell-vm/runtime/README.md @@ -19,7 +19,7 @@ that enables these networking and sandboxing features. ## Directory Structure -``` +```text runtime/ kernel/ openshell.kconfig # Kernel config fragment (networking + sandboxing) @@ -29,7 +29,7 @@ runtime/ Each platform builds its own kernel and runtime natively. -``` +```text Linux ARM64: builds aarch64 kernel -> .so (parallel) Linux AMD64: builds x86_64 kernel -> .so (parallel) macOS ARM64: builds aarch64 kernel -> .dylib @@ -67,7 +67,7 @@ FROM_SOURCE=1 mise run vm:setup Build artifacts are placed in `target/libkrun-build/`: -``` +```text target/libkrun-build/ libkrun.so / libkrun.dylib # The VMM library libkrunfw.so* / libkrunfw.dylib # Kernel firmware library @@ -89,7 +89,7 @@ subsystem directly and avoids this entirely. At VM boot, the openshell-vm binary logs provenance information about the loaded runtime: -``` +```text runtime: /path/to/openshell-vm.runtime libkrunfw: libkrunfw.dylib sha256: a1b2c3d4e5f6... @@ -100,7 +100,8 @@ runtime: /path/to/openshell-vm.runtime ``` For stock runtimes: -``` + +```text runtime: /path/to/openshell-vm.runtime libkrunfw: libkrunfw.dylib sha256: f6e5d4c3b2a1... @@ -140,6 +141,7 @@ mise run vm ### "FailedCreatePodSandBox" bridge errors The kernel does not have bridge support. Verify: + ```bash # Inside VM: ip link add test0 type bridge && echo "bridge OK" && ip link del test0 @@ -150,6 +152,7 @@ If this fails, you are running the stock runtime. Build and use the custom one. ### kube-proxy CrashLoopBackOff kube-proxy runs in nftables mode. If it crashes, verify nftables support: + ```bash # Inside VM: nft list ruleset @@ -158,6 +161,7 @@ nft list ruleset If this fails, the kernel may lack `CONFIG_NF_TABLES`. Use the custom runtime. Common errors: + - `unknown option "--xor-mark"`: kube-proxy is running in iptables mode instead of nftables. Verify `--kube-proxy-arg=proxy-mode=nftables` is in the k3s args. @@ -165,12 +169,14 @@ Common errors: If libkrunfw is updated (e.g., via `brew upgrade`), the stock runtime may change. Check provenance: + ```bash # Look for provenance info in VM boot output grep "runtime:" ~/.local/share/openshell/openshell-vm/console.log ``` Re-build the custom runtime if needed: + ```bash FROM_SOURCE=1 mise run vm:setup mise run vm:build diff --git a/docs/.markdownlint-cli2.jsonc b/docs/.markdownlint-cli2.jsonc new file mode 100644 index 0000000000..6e86e62867 --- /dev/null +++ b/docs/.markdownlint-cli2.jsonc @@ -0,0 +1,8 @@ +{ + "config": { + // MDX pages get their title from Fern frontmatter, not a top-level H1. + "MD041": false, + // MDX uses JSX components (, , ...) that look like HTML. + "MD033": false + } +} diff --git a/docs/CONTRIBUTING.mdx b/docs/CONTRIBUTING.mdx index 1fcaf322ac..f2426873a7 100644 --- a/docs/CONTRIBUTING.mdx +++ b/docs/CONTRIBUTING.mdx @@ -56,13 +56,15 @@ PRs that touch `docs/**` or `fern/**` are validated by `.github/workflows/branch - Published docs use Fern MDX under `docs/`. - Every page starts with YAML frontmatter. Use `title` and `description` on every page, then add page-level metadata like `sidebar-title`, `keywords`, and `position` when the page needs them. - Include the SPDX license header as YAML comments inside frontmatter: - ``` + + ```text --- # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 title: "Page Title" --- ``` + - Do not repeat the page title as a body H1. Fern renders the title from frontmatter. ### Frontmatter Template @@ -122,9 +124,11 @@ These patterns are common in LLM-generated text and erode trust with technical r - End every sentence with a period. - Use `code` formatting for CLI commands, file paths, flags, parameter names, and values. - Use `shell` code blocks for copyable CLI examples. Do not prefix commands with `$`: + ```shell openshell gateway start ``` + - Use `text` code blocks for transcripts, log output, and examples that should not be copied verbatim. - Use tables for structured comparisons. Keep tables simple (no nested formatting). - Use Fern components like ``, ``, and `` for callouts, not bold text. @@ -157,13 +161,13 @@ Use these consistently: 5. Run `mise run pre-commit` to catch formatting issues. 6. Open a PR with `docs:` as the conventional commit type. -``` +```text docs: update gateway deployment instructions ``` If your doc change accompanies a code change, include both in the same PR and use the code change's commit type: -``` +```text feat(cli): add --gpu flag to gateway start ``` diff --git a/docs/get-started/tutorials/github-sandbox.mdx b/docs/get-started/tutorials/github-sandbox.mdx index 8492dde715..ee3e16759d 100644 --- a/docs/get-started/tutorials/github-sandbox.mdx +++ b/docs/get-started/tutorials/github-sandbox.mdx @@ -152,7 +152,7 @@ The following steps outline the expected process done by the agent: 1. Inspects the deny reasons. 2. Writes an updated policy that adds `github_git` and `github_api` blocks that grant write access to your repository. -3. Saves the policy to `/tmp/sandbox-policy-update.yaml`. +3. Saves the policy to `/tmp/sandbox-policy-update.yaml`. ## Review the Generated Policy diff --git a/docs/get-started/tutorials/inference-ollama.mdx b/docs/get-started/tutorials/inference-ollama.mdx index a123a2b28a..4fcc018a7b 100644 --- a/docs/get-started/tutorials/inference-ollama.mdx +++ b/docs/get-started/tutorials/inference-ollama.mdx @@ -45,7 +45,7 @@ Chat with a local model ollama run qwen3.5 ``` -Or a cloud model +Or a cloud model ```shell ollama run kimi-k2.5:cloud @@ -77,7 +77,7 @@ ollama launch claude --yes --model qwen3.5 | No local GPU | `qwen3.5:cloud` | Runs on Ollama's cloud infrastructure, no `ollama pull` required | -Cloud models use the `:cloud` tag suffix and do not require local hardware. +Cloud models use the `:cloud` tag suffix and do not require local hardware. ```shell openshell sandbox create --from ollama diff --git a/docs/get-started/tutorials/local-inference-lmstudio.mdx b/docs/get-started/tutorials/local-inference-lmstudio.mdx index 7b87df2e5e..976770d1fe 100644 --- a/docs/get-started/tutorials/local-inference-lmstudio.mdx +++ b/docs/get-started/tutorials/local-inference-lmstudio.mdx @@ -43,6 +43,7 @@ irm https://lmstudio.ai/install.ps1 | iex And start llmster: + ```shell lms daemon up ``` @@ -64,6 +65,7 @@ If you're using llmster in headless mode, run `lms server start --bind 0.0.0.0`. In the LM Studio app, head to the Model Search tab to download a small model like Qwen3.5 2B. In the terminal, use the following command to download and load the model: + ```shell lms get qwen/qwen3.5-2b lms load qwen/qwen3.5-2b diff --git a/docs/index.mdx b/docs/index.mdx index 05011e5557..73b8d1d0b1 100644 --- a/docs/index.mdx +++ b/docs/index.mdx @@ -9,7 +9,6 @@ position: 1 import { BadgeLinks } from "./_components/BadgeLinks"; - with { } breaks MDX/acorn */} +{/*Terminal demo styles live in fern/main.css — inline