From 5f38b7c42e918f4dc2f7d56206692e39bab5ef8e Mon Sep 17 00:00:00 2001 From: Ian Miller <75687988+r3v5@users.noreply.github.com> Date: Fri, 10 Jul 2026 16:37:30 +0100 Subject: [PATCH 01/13] fix(tui): route warning logs to status bar instead of stderr (#2210) * fix(tui): route warning logs to status bar instead of stderr tracing::warn/info/debug calls in the TUI crate write to stderr via the global tracing subscriber. In ratatui's alternate-screen/raw-mode, stderr writes corrupt the terminal layout. Error-state sandboxes amplify this as background gRPC polls fail every 2s tick. Replace all 25 tracing calls with app.status_text assignments for direct-access sites, and Vec accumulation for the spawned start_port_forwards task. Add ForwardWarnings event variant to decouple forward warning delivery from the sandbox name in CreateResult, preventing downstream gRPC lookup failures. Closes #2120 Signed-off-by: Ian Miller * feat(tui): add ForwardWarnings event variant New event type for non-fatal port-forward warnings during sandbox creation. Keeps warning delivery separate from the sandbox name in CreateResult to avoid corrupting downstream gRPC lookups. Signed-off-by: Ian Miller * chore(tui): remove tracing dependency Compile-time guard against reintroducing stderr-writing tracing calls in the TUI crate. 14 other crates retain the dependency. Signed-off-by: Ian Miller --------- Signed-off-by: Ian Miller --- Cargo.lock | 1 - crates/openshell-tui/Cargo.toml | 1 - crates/openshell-tui/src/event.rs | 2 + crates/openshell-tui/src/lib.rs | 102 ++++++++++++++++-------------- 4 files changed, 58 insertions(+), 48 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 13b670f558..916969ac5f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4018,7 +4018,6 @@ dependencies = [ "terminal-colorsaurus", "tokio", "tonic", - "tracing", "url", ] diff --git a/crates/openshell-tui/Cargo.toml b/crates/openshell-tui/Cargo.toml index 2381661364..caa2ec1ab4 100644 --- a/crates/openshell-tui/Cargo.toml +++ b/crates/openshell-tui/Cargo.toml @@ -25,7 +25,6 @@ tonic = { workspace = true, features = ["tls-native-roots"] } miette = { workspace = true } owo-colors = { workspace = true } serde = { workspace = true } -tracing = { workspace = true } url = { workspace = true } [lints] diff --git a/crates/openshell-tui/src/event.rs b/crates/openshell-tui/src/event.rs index 4c6eeb8b46..6964ad4ac2 100644 --- a/crates/openshell-tui/src/event.rs +++ b/crates/openshell-tui/src/event.rs @@ -51,6 +51,8 @@ pub enum Event { SandboxSettingSetResult(Result), /// Sandbox setting delete result: `Ok(revision)` or `Err(message)`. SandboxSettingDeleteResult(Result), + /// Non-fatal warnings from port-forward setup after sandbox creation. + ForwardWarnings(Vec), } pub struct EventHandler { diff --git a/crates/openshell-tui/src/lib.rs b/crates/openshell-tui/src/lib.rs index 7992666d38..da77dc10c3 100644 --- a/crates/openshell-tui/src/lib.rs +++ b/crates/openshell-tui/src/lib.rs @@ -235,7 +235,7 @@ pub async fn run( app.apply_global_settings(settings, revision); } Err(msg) => { - tracing::warn!("failed to fetch global settings: {msg}"); + app.status_text = format!("failed to fetch global settings: {msg}"); } }, Some(Event::GlobalSettingSetResult(result)) => { @@ -285,6 +285,9 @@ pub async fn run( } fetch_sandbox_detail(&mut app).await; } + Some(Event::ForwardWarnings(warnings)) => { + app.status_text = format!("port forward issues: {}", warnings.join("; ")); + } Some(Event::Mouse(mouse)) => match mouse.kind { MouseEventKind::ScrollUp if app.focus == Focus::SandboxLogs => { app.scroll_logs(-3); @@ -722,10 +725,14 @@ async fn handle_sandbox_delete(app: &mut App) { }; // Stop any active port forwards before deleting (mirrors CLI behavior). - if let Ok(stopped) = openshell_core::forward::stop_forwards_for_sandbox(&sandbox_name) { - for port in &stopped { - tracing::info!("stopped forward of port {port} for sandbox {sandbox_name}"); - } + if let Ok(stopped) = openshell_core::forward::stop_forwards_for_sandbox(&sandbox_name) + && !stopped.is_empty() + { + let ports: Vec = stopped.iter().map(ToString::to_string).collect(); + app.status_text = format!( + "stopped port forwards [{}] for sandbox {sandbox_name}", + ports.join(", ") + ); } let req = openshell_core::proto::DeleteSandboxRequest { name: sandbox_name }; @@ -777,11 +784,11 @@ async fn fetch_sandbox_detail(app: &mut App) { } } Ok(Err(e)) => { - tracing::warn!("failed to fetch sandbox detail: {}", e.message()); + app.status_text = format!("failed to fetch sandbox detail: {}", e.message()); None } Err(_) => { - tracing::warn!("sandbox detail request timed out"); + app.status_text = "sandbox detail request timed out".to_string(); None } }; @@ -812,10 +819,10 @@ async fn fetch_sandbox_detail(app: &mut App) { app.apply_sandbox_settings(inner.settings); } Ok(Err(e)) => { - tracing::warn!("failed to fetch sandbox policy: {}", e.message()); + app.status_text = format!("failed to fetch sandbox policy: {}", e.message()); } Err(_) => { - tracing::warn!("sandbox policy request timed out"); + app.status_text = "sandbox policy request timed out".to_string(); } } } @@ -1419,7 +1426,7 @@ fn spawn_create_sandbox(app: &mut App, tx: mpsc::UnboundedSender) { // Start port forwards if requested. if !ports.is_empty() { - start_port_forwards( + let forward_warnings = start_port_forwards( &mut client, &endpoint, &gateway_name, @@ -1428,6 +1435,9 @@ fn spawn_create_sandbox(app: &mut App, tx: mpsc::UnboundedSender) { &ports, ) .await; + if !forward_warnings.is_empty() { + let _ = tx.send(Event::ForwardWarnings(forward_warnings)); + } } } @@ -1448,7 +1458,9 @@ async fn start_port_forwards( sandbox_name: &str, sandbox_id: &str, specs: &[openshell_core::forward::ForwardSpec], -) { +) -> Vec { + let mut warnings = Vec::new(); + // Create SSH session. let session = { let req = openshell_core::proto::CreateSshSessionRequest { @@ -1457,18 +1469,20 @@ async fn start_port_forwards( match tokio::time::timeout(Duration::from_secs(10), client.create_ssh_session(req)).await { Ok(Ok(resp)) => resp.into_inner(), Ok(Err(e)) => { - tracing::warn!("SSH session failed for forwards: {}", e.message()); - return; + warnings.push(format!("SSH session failed for forwards: {}", e.message())); + return warnings; } Err(_) => { - tracing::warn!("SSH session timed out for forwards"); - return; + warnings.push("SSH session timed out for forwards".to_string()); + return warnings; } } }; if let Err(err) = validate_ssh_session_response(&session) { - tracing::warn!("gateway returned invalid SSH session response for forwards: {err}"); - return; + warnings.push(format!( + "gateway returned invalid SSH session response for forwards: {err}" + )); + return warnings; } // Resolve gateway address. @@ -1482,8 +1496,8 @@ async fn start_port_forwards( let exe = match std::env::current_exe() { Ok(p) => p, Err(e) => { - tracing::warn!("failed to find executable for forwards: {e}"); - return; + warnings.push(format!("failed to find executable for forwards: {e}")); + return warnings; } }; let proxy_command = build_proxy_command( @@ -1563,16 +1577,18 @@ async fn start_port_forwards( } } Ok(Ok(false)) => { - tracing::warn!("SSH forward exited with error for port {port_val}"); + warnings.push(format!("SSH forward exited with error for port {port_val}")); } Ok(Err(e)) => { - tracing::warn!("forward failed for port {port_val}: {e}"); + warnings.push(format!("forward failed for port {port_val}: {e}")); } Err(e) => { - tracing::warn!("forward task panicked for port {port_val}: {e}"); + warnings.push(format!("forward task panicked for port {port_val}: {e}")); } } } + + warnings } // --------------------------------------------------------------------------- @@ -1927,11 +1943,11 @@ async fn refresh_providers(app: &mut App) { .map(|profile| (profile.id.clone(), profile)) .collect::>(), Ok(Err(e)) => { - tracing::warn!("failed to list provider profiles: {}", e.message()); + app.status_text = format!("failed to list provider profiles: {}", e.message()); HashMap::new() } Err(_) => { - tracing::warn!("list provider profiles timed out"); + app.status_text = "list provider profiles timed out".to_string(); HashMap::new() } } @@ -1946,10 +1962,10 @@ async fn refresh_providers(app: &mut App) { let result = tokio::time::timeout(Duration::from_secs(5), app.client.list_providers(req)).await; match result { Ok(Err(e)) => { - tracing::warn!("failed to list providers: {}", e.message()); + app.status_text = format!("failed to list providers: {}", e.message()); } Err(_) => { - tracing::warn!("list providers timed out"); + app.status_text = "list providers timed out".to_string(); } Ok(Ok(resp)) => { let providers = resp.into_inner().providers; @@ -1994,10 +2010,10 @@ async fn refresh_global_settings(app: &mut App) { tokio::time::timeout(Duration::from_secs(5), app.client.get_gateway_config(req)).await; match result { Ok(Err(e)) => { - tracing::warn!("failed to fetch global settings: {}", e.message()); + app.status_text = format!("failed to fetch global settings: {}", e.message()); } Err(_) => { - tracing::warn!("get gateway settings timed out"); + app.status_text = "get gateway settings timed out".to_string(); } Ok(Ok(resp)) => { let inner = resp.into_inner(); @@ -2275,10 +2291,10 @@ async fn refresh_sandboxes(app: &mut App) { let result = tokio::time::timeout(Duration::from_secs(5), app.client.list_sandboxes(req)).await; match result { Ok(Err(e)) => { - tracing::warn!("failed to list sandboxes: {}", e.message()); + app.status_text = format!("failed to list sandboxes: {}", e.message()); } Err(_) => { - tracing::warn!("list sandboxes timed out"); + app.status_text = "list sandboxes timed out".to_string(); } Ok(Ok(resp)) => { let sandboxes = resp.into_inner().sandboxes; @@ -2387,10 +2403,10 @@ async fn refresh_sandbox_policy(app: &mut App) { app.apply_sandbox_settings(inner.settings); } Ok(Err(e)) => { - tracing::warn!("failed to refresh sandbox policy: {}", e.message()); + app.status_text = format!("failed to refresh sandbox policy: {}", e.message()); } Err(_) => { - tracing::warn!("sandbox policy refresh timed out"); + app.status_text = "sandbox policy refresh timed out".to_string(); } } } @@ -2406,20 +2422,14 @@ async fn refresh_draft_chunks(app: &mut App) { status_filter: String::new(), }; - match tokio::time::timeout(Duration::from_secs(5), app.client.get_draft_policy(req)).await { - Ok(Ok(resp)) => { - let inner = resp.into_inner(); - app.draft_chunks = inner.chunks; - app.draft_version = inner.draft_version; - if app.draft_selected >= app.draft_chunks.len() && !app.draft_chunks.is_empty() { - app.draft_selected = app.draft_chunks.len() - 1; - } - } - Ok(Err(e)) => { - tracing::debug!("draft chunks refresh: {}", e.message()); - } - Err(_) => { - tracing::debug!("draft chunks refresh timed out"); + if let Ok(Ok(resp)) = + tokio::time::timeout(Duration::from_secs(5), app.client.get_draft_policy(req)).await + { + let inner = resp.into_inner(); + app.draft_chunks = inner.chunks; + app.draft_version = inner.draft_version; + if app.draft_selected >= app.draft_chunks.len() && !app.draft_chunks.is_empty() { + app.draft_selected = app.draft_chunks.len() - 1; } } } From ccdac9cec49af1f15afaaa1dcdc694bedab040c3 Mon Sep 17 00:00:00 2001 From: Kirit Thadaka Date: Fri, 10 Jul 2026 08:38:26 -0700 Subject: [PATCH 02/13] fix(mcp): include tool names in policy logs (#2189) Signed-off-by: Kirit93 --- crates/openshell-ocsf/src/format/shorthand.rs | 70 ++++++++++++++++++- .../src/l7/relay.rs | 47 ++++++++++++- 2 files changed, 113 insertions(+), 4 deletions(-) diff --git a/crates/openshell-ocsf/src/format/shorthand.rs b/crates/openshell-ocsf/src/format/shorthand.rs index 3143b6d68e..0cb5d906c3 100644 --- a/crates/openshell-ocsf/src/format/shorthand.rs +++ b/crates/openshell-ocsf/src/format/shorthand.rs @@ -218,7 +218,19 @@ impl OcsfEvent { (false, true) => format!(" {action}"), (false, false) => format!(" {action}{arrow}"), }; - format!("HTTP:{method} {sev}{detail}{rule_ctx}{reason_ctx}") + // Denied HTTP events surface their message through reason_ctx. + // Allowed MCP decisions also need the JSON-RPC message so the + // selected tool name is visible in the shorthand log stream. + let message_ctx = if action != "DENIED" + && e.firewall_rule + .as_ref() + .is_some_and(|rule| rule.rule_type == "l7-mcp") + { + message_tag(&e.base) + } else { + String::new() + }; + format!("HTTP:{method} {sev}{detail}{rule_ctx}{reason_ctx}{message_ctx}") } Self::SshActivity(e) => { @@ -534,6 +546,62 @@ mod tests { ); } + #[test] + fn test_http_activity_shorthand_mcp_shows_tool_for_allow_and_deny() { + let mut event_base = base(4002, "HTTP Activity", 4, "Network Activity", 0, "Other"); + event_base.set_message( + "JSONRPC_L7_REQUEST decision=allow rule_methods=tools/call tools=move_head", + ); + let event = OcsfEvent::HttpActivity(HttpActivityEvent { + base: event_base, + http_request: Some(HttpRequest::new( + "POST", + Url::new("http", "host.openshell.internal", "/mcp", 8766), + )), + http_response: None, + src_endpoint: None, + dst_endpoint: None, + proxy_endpoint: None, + actor: None, + firewall_rule: Some(FirewallRule::new("reachy-mcp", "l7-mcp")), + action: Some(ActionId::Allowed), + disposition: Some(DispositionId::Allowed), + observation_point_id: None, + is_src_dst_assignment_known: None, + }); + + let shorthand = event.format_shorthand(); + assert!(shorthand.contains("engine:l7-mcp")); + assert!(shorthand.contains("tools=move_head")); + + let mut denied_base = base(4002, "HTTP Activity", 4, "Network Activity", 0, "Other"); + denied_base.severity = crate::enums::SeverityId::Medium; + denied_base.set_message( + "JSONRPC_L7_REQUEST decision=deny rule_methods=tools/call tools=move_head", + ); + let denied_event = OcsfEvent::HttpActivity(HttpActivityEvent { + base: denied_base, + http_request: Some(HttpRequest::new( + "POST", + Url::new("http", "host.openshell.internal", "/mcp", 8766), + )), + http_response: None, + src_endpoint: None, + dst_endpoint: None, + proxy_endpoint: None, + actor: None, + firewall_rule: Some(FirewallRule::new("reachy-mcp", "l7-mcp")), + action: Some(ActionId::Denied), + disposition: Some(DispositionId::Blocked), + observation_point_id: None, + is_src_dst_assignment_known: None, + }); + + let denied_shorthand = denied_event.format_shorthand(); + assert!(denied_shorthand.contains("[reason:")); + assert!(denied_shorthand.contains("tools=move_head")); + } + #[test] fn test_network_activity_shorthand_denied_shows_reason() { let mut b = base(4001, "Network Activity", 4, "Network Activity", 1, "Open"); diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index ed2bde1135..c43fb35f9d 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -571,7 +571,11 @@ fn l7_protocol_log_summary( } if let Some(info) = jsonrpc_info { - return format!(" rule_methods={}", rule_method_names_for_log(info)); + return format!( + " rule_methods={} tools={}", + rule_method_names_for_log(info), + tool_names_for_log(info) + ); } String::new() @@ -1428,8 +1432,9 @@ pub(crate) fn jsonrpc_log_message( reason: &str, ) -> String { let rule_methods = rule_method_names_for_log(info); + let tools = tool_names_for_log(info); format!( - "JSONRPC_L7_REQUEST decision={decision} http_method={http_method} endpoint={endpoint} rule_methods={rule_methods} policy_version={policy_version} reason={reason}" + "JSONRPC_L7_REQUEST decision={decision} rule_methods={rule_methods} tools={tools} http_method={http_method} endpoint={endpoint} policy_version={policy_version} reason={reason}" ) } @@ -1444,6 +1449,20 @@ pub(crate) fn rule_method_names_for_log(info: &crate::l7::jsonrpc::JsonRpcReques .join(",") } +pub(crate) fn tool_names_for_log(info: &crate::l7::jsonrpc::JsonRpcRequestInfo) -> String { + let tools = info + .calls + .iter() + .filter_map(|call| call.tool.as_deref()) + .map(sanitize_log_token) + .collect::>(); + if tools.is_empty() { + "-".to_string() + } else { + tools.join(",") + } +} + fn sanitize_log_token(value: &str) -> String { value .chars() @@ -2538,7 +2557,6 @@ network_policies: let (allowed, reason) = evaluate_l7_request(&tunnel_engine, &ctx, &request).unwrap(); assert!(allowed, "{reason}"); - request.jsonrpc = Some(crate::l7::jsonrpc::parse_jsonrpc_body( br#"{"jsonrpc":"2.0","id":1,"method":"reports.search","params":["ignored",{"nested":true}]}"#, crate::l7::jsonrpc::JsonRpcInspectionMode::JsonRpc, @@ -2604,6 +2622,17 @@ network_policies: let (allowed, reason) = evaluate_l7_request(&tunnel_engine, &ctx, &request).unwrap(); assert!(allowed, "{reason}"); + let allowed_info = request.jsonrpc.as_ref().expect("parsed MCP request"); + let allowed_message = jsonrpc_log_message( + "allow", + "POST", + "api.example.test:443/mcp", + allowed_info, + 42, + &reason, + ); + assert!(allowed_message.contains("rule_methods=tools/call")); + assert!(allowed_message.contains("tools=read_status")); request.jsonrpc = Some(crate::l7::jsonrpc::parse_jsonrpc_body( br#"{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"delete_resource","arguments":{"scope":"workspace/main"}}}"#, @@ -2625,6 +2654,17 @@ network_policies: reason.contains("deny rule"), "deny reason should identify policy denial: {reason}" ); + let denied_message = jsonrpc_log_message( + "deny", + "POST", + "api.example.test:443/mcp", + parsed, + 42, + &reason, + ); + assert!(denied_message.contains("rule_methods=tools/call")); + assert!(denied_message.contains("tools=delete_resource")); + assert!(!denied_message.contains("workspace/main")); } #[test] @@ -2644,6 +2684,7 @@ network_policies: assert!(message.contains("endpoint=jsonrpc.example.com:443/rpc")); assert!(message.contains("rule_methods=reports.archive")); + assert!(message.contains("tools=-")); assert!(message.contains("policy_version=42")); assert!(!message.contains("delete_resource")); assert!(!message.contains("secret-scope")); From caaa51653701854941192cef01a8e05088cc1b89 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Jul 2026 15:42:13 +0000 Subject: [PATCH 03/13] chore(deps): bump astral-sh/setup-uv from 8.3.1 to 8.3.2 (#2206) Bumps [astral-sh/setup-uv](https://github.com/astral-sh/setup-uv) from 8.3.1 to 8.3.2. - [Release notes](https://github.com/astral-sh/setup-uv/releases) - [Commits](https://github.com/astral-sh/setup-uv/compare/f98e06938123ccabd21905ea5d0069192241f9f1...11f9893b081a58869d3b5fccaea48c9e9e46f990) --- updated-dependencies: - dependency-name: astral-sh/setup-uv dependency-version: 8.3.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/sync-docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync-docs.yml b/.github/workflows/sync-docs.yml index c1ca4b39be..f84f4cdfef 100644 --- a/.github/workflows/sync-docs.yml +++ b/.github/workflows/sync-docs.yml @@ -93,7 +93,7 @@ jobs: path: docs-website - name: Install uv - uses: astral-sh/setup-uv@f98e06938123ccabd21905ea5d0069192241f9f1 # v8.3.1 + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 with: version: "0.10.12" From 8c0ecac8cfad2ddcbe6d9ab1c0132df90503221b Mon Sep 17 00:00:00 2001 From: Christian Zaccaria <73656840+ChristianZaccaria@users.noreply.github.com> Date: Fri, 10 Jul 2026 16:45:56 +0100 Subject: [PATCH 04/13] docs(openshift): simplify install steps and add Helm README entries for OpenShift overrides (#2125) Signed-off-by: ChristianZaccaria --- deploy/helm/openshell/README.md | 2 ++ deploy/helm/openshell/README.md.gotmpl | 2 ++ docs/kubernetes/openshift.mdx | 12 +++--------- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index f3a86f884f..6bc459567b 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -25,6 +25,8 @@ helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version -The OpenShift install path is experimental. It currently requires running sandbox pods under the `privileged` SCC and installing the gateway with TLS and the PKI init job disabled. Use only for evaluation on a private network. +The OpenShift install path is experimental. It currently requires running sandbox pods under the `privileged` SCC and installing the gateway with TLS disabled. Use only for evaluation on a private network. OpenShift's [Security Context Constraints](https://docs.openshift.com/container-platform/latest/authentication/managing-security-context-constraints.html) reject the chart's default pod security settings. Installing on OpenShift requires precreating the namespace, granting the `privileged` SCC to the sandbox service account, and overriding a few chart values so the cluster admission controller can assign UIDs and FS groups itself. @@ -46,7 +46,6 @@ oc adm policy add-scc-to-user privileged -z openshell-sandbox -n openshell helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart \ --version \ --namespace openshell \ - --set pkiInitJob.enabled=false \ --set server.disableTls=true \ --set podSecurityContext.fsGroup=null \ --set securityContext.runAsUser=null @@ -54,14 +53,9 @@ helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart \ | Override | Reason | |---|---| -| `pkiInitJob.enabled=false` | Skips the built-in TLS PKI Job. TLS must also be disabled unless you provide TLS Secrets another way. | -| `server.disableTls=true` | The gateway has no certificates without `pkiInitJob`, so it must run plaintext. | +| `server.disableTls=true` | Runs the gateway over plaintext HTTP for simpler evaluation. | | `podSecurityContext.fsGroup=null` / `securityContext.runAsUser=null` | Clear the chart's hardcoded UID and fsGroup so OpenShift's SCC admission can assign them. | -The gateway still needs the sandbox JWT signing Secret. When disabling -`pkiInitJob` without enabling cert-manager, pre-create that Secret before -installing the chart. - ## Wait for the gateway to be ready ```shell @@ -90,6 +84,6 @@ openshell status ## Next Steps -- For TLS-enabled deployments, refer to [Managing Certificates](/kubernetes/managing-certificates) after SCC-compatible PKI is supported. +- For TLS-enabled deployments, refer to [Managing Certificates](/kubernetes/managing-certificates). - To expose the gateway externally, refer to [Ingress](/kubernetes/ingress). - To configure OIDC authentication, refer to [Access Control](/kubernetes/access-control). From 233d207e70170fd3c5770d3547b62733445ccfe2 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 10 Jul 2026 18:51:39 +0200 Subject: [PATCH 05/13] docs(issues): require release and duplicate checks (#2214) Signed-off-by: Evan Lezar --- .agents/skills/create-github-issue/SKILL.md | 13 ++++++--- .agents/skills/triage-issue/SKILL.md | 29 ++++++++++++++++----- .github/ISSUE_TEMPLATE/bug_report.yml | 15 +++++++++-- 3 files changed, 46 insertions(+), 11 deletions(-) diff --git a/.agents/skills/create-github-issue/SKILL.md b/.agents/skills/create-github-issue/SKILL.md index b1311a297e..d196fc8c8d 100644 --- a/.agents/skills/create-github-issue/SKILL.md +++ b/.agents/skills/create-github-issue/SKILL.md @@ -17,7 +17,7 @@ This project uses YAML form issue templates. When creating issues, match the tem ### Bug Reports -Do not add a type label automatically. The body must include an **Agent Diagnostic** section — this is required by the template and enforced by project convention. Apply area or topic labels only when they are clearly known. +Do not add a type label automatically. The body must include an **Agent Diagnostic** section — this is required by the template and enforced by project convention. The diagnostic must identify the OpenShell version tested, whether the latest release or known fixes were checked, and whether possible duplicate issues were searched. If the agent cannot verify the latest release or search existing issues, say so explicitly instead of guessing. Apply area or topic labels only when they are clearly known. ```bash gh issue create \ @@ -25,8 +25,13 @@ gh issue create \ --body "$(cat <<'EOF' ## Agent Diagnostic - +- Skills loaded: +- OpenShell version tested: +- Latest release checked: +- Known fixes reviewed: +- Possible duplicates reviewed: +- Findings: +- Remaining reason for filing: ## Description @@ -44,6 +49,8 @@ What was found? What was tried?> - OS: - Docker: - OpenShell: +- Latest release checked: +- Possible duplicates checked: ## Logs diff --git a/.agents/skills/triage-issue/SKILL.md b/.agents/skills/triage-issue/SKILL.md index 54e726a5d9..8d602a9023 100644 --- a/.agents/skills/triage-issue/SKILL.md +++ b/.agents/skills/triage-issue/SKILL.md @@ -51,7 +51,7 @@ Query all open issues with the `state:triage-needed` label and process them in s gh issue list --label "state:triage-needed" --state open --json number,title --jq '.[].number' ``` -For each issue returned, run the full triage workflow (Steps 1-6). Report a summary at the end listing each issue and its classification. +For each issue returned, run the full triage workflow (Steps 1-7). Report a summary at the end listing each issue and its classification. ## Step 1: Fetch the Issue @@ -89,7 +89,23 @@ Check whether the issue body contains a substantive agent diagnostic section. Lo **If the diagnostic section is substantive**, proceed to Step 4. -## Step 4: Diagnose and Validate +## Step 4: Check Reported Version and Known Fixes + +Before deeper diagnosis, determine whether the report may already be fixed in a newer release. + +1. Extract the reported OpenShell version from the issue body, Agent Diagnostic, environment section, logs, and comments. If no version is provided, record that as missing context and continue. +2. Check current release information and known fixes when available: + - `gh release list --limit 10` + - `gh release view ` + - linked issues, merged PRs, release notes, and local git tags/history +3. If network access or release metadata is unavailable, state the limitation in the triage comment instead of guessing. + +If the issue targets an older OpenShell release and a newer release or merged PR appears to address the same behavior: + +- If the reporter has already reproduced the issue on the fixed/current release, continue to Step 5. +- If the reporter has not tested the fixed/current release, use the `fixed-in-release` classification in Step 6. Reference the fixing version and PR/issue when known, and ask for a fresh report or reopen if the issue still reproduces on that version. + +## Step 5: Diagnose and Validate Assess the report by investigating the codebase. Use the `principal-engineer-reviewer` sub-agent via the Task tool: @@ -114,7 +130,7 @@ Based on the sub-agent's analysis, also attempt to validate the report directly: - For inference and provider-topology issues: reference the `debug-inference` skill's known failure patterns - For CLI/usage issues: reference the `openshell-cli` skill's command reference -## Step 5: Classify +## Step 6: Classify Based on the investigation, classify the issue into one of these categories: @@ -122,12 +138,13 @@ Based on the investigation, classify the issue into one of these categories: |---------------|----------|--------| | **bug-confirmed** | Agent diagnostic and codebase analysis confirm a real defect | Apply relevant `area:*` or `topic:*` labels as needed, remove `state:triage-needed`, and assign the built-in `Bug` issue type manually if needed | | **feature-valid** | Design proposal is sound, feasible given the architecture | Apply relevant `area:*` or `topic:*` labels as needed, remove `state:triage-needed`, and assign the built-in `Feature` issue type manually if needed | +| **fixed-in-release** | Report targets an older OpenShell release and a newer release or merged PR appears to address the behavior; no fixed/current-release reproduction is provided | Comment with the fixing version and PR/issue when known. Close as completed when the fix is clear, or request a retest if confirmation is still needed. Remove `state:triage-needed` when closing | | **duplicate** | An existing open issue covers this | Link the duplicate, close with comment | | **user-error** | The reported behavior is expected, or the issue is a misconfiguration | Comment with explanation and guidance, close | | **needs-more-info** | Report is substantive but missing critical reproduction details | Comment requesting specifics, keep `state:triage-needed` | | **needs-investigation** | Report appears valid but requires deeper analysis (spike candidate) | Label `spike`, remove `state:triage-needed` | -## Step 6: Post Triage Comment +## Step 7: Post Triage Comment Post a structured comment with the triage marker: @@ -136,7 +153,7 @@ Post a structured comment with the triage marker: > > ## Triage Assessment > -> **Classification:** +> **Classification:** > > ### Summary > <2-3 sentences: what was found, whether the report is valid> @@ -148,7 +165,7 @@ Post a structured comment with the triage marker: > ``` -Apply the appropriate labels as determined in Step 5. +Apply the appropriate labels as determined in Step 6. **Do not apply `state:agent-ready`.** That is always a human decision. diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 74fa4bf5d4..de5d8112ae 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -9,15 +9,20 @@ body: OpenShell is an agent-first project. Before filing this bug, point your coding agent at the repo and have it investigate using the available skills (`debug-openshell-cluster`, `debug-inference`, `openshell-cli`, etc.). See [CONTRIBUTING.md](https://github.com/NVIDIA/OpenShell/blob/main/CONTRIBUTING.md) for the full skills table. + Please also check the latest OpenShell release and search existing issues for possible duplicates before filing. If you cannot upgrade, retest, or search existing issues, explain why in the diagnostic. + - type: textarea id: agent-diagnostic attributes: label: Agent Diagnostic description: | - Paste the output from your agent's investigation of this bug. What skills did it load? What did it find? What did it try? + Paste the output from your agent's investigation of this bug. What skills did it load? What did it find? What did it try? Which OpenShell version did it test, and did it check the latest release, known fixes, and possible duplicates? placeholder: | Example: - Loaded `debug-inference` skill + - Tested OpenShell v0.x.x + - Checked latest release / known fixes: no matching fix found + - Searched existing issues for duplicates: no matching issue found - Ran `openshell inference get` and `openshell provider get ollama` - Found `OPENAI_BASE_URL=http://127.0.0.1:11434/v1`, which is unreachable from the gateway - Updated the provider to use `host.openshell.internal`, but the issue persists because the gateway is remote @@ -47,11 +52,13 @@ body: id: environment attributes: label: Environment - description: OS, Docker version, OpenShell version, and any other relevant details. + description: OS, Docker version, OpenShell version tested, whether the latest release and existing issues were checked, and any other relevant details. placeholder: | - OS: macOS 15.2 / Ubuntu 24.04 / Windows 11 + WSL2 - Docker: Docker Desktop 4.x / Docker Engine 27.x - OpenShell: v0.x.x (output of `openshell --version`) + - Latest release checked: yes / no, because ... + - Possible duplicates checked: yes / no, because ... validations: required: true @@ -75,5 +82,9 @@ body: required: true - label: I loaded relevant skills (e.g., `debug-openshell-cluster`, `debug-inference`, `openshell-cli`) required: true + - label: I checked the latest OpenShell release and either reproduced the issue there or explained why I cannot upgrade/test it + required: true + - label: I searched existing issues for possible duplicates or explained why I could not + required: true - label: My agent could not resolve this — the diagnostic above explains why required: true From 10702133a30e9c6cc621e8ea241171b518c17296 Mon Sep 17 00:00:00 2001 From: Florent BENOIT Date: Fri, 10 Jul 2026 20:26:49 +0200 Subject: [PATCH 06/13] fix(core): pin supervisor image tag to gateway version for all drivers (#2070) * fix(core): pin supervisor image tag to gateway version for all drivers The Podman and Kubernetes drivers defaulted the supervisor image to `:latest` via DEFAULT_SUPERVISOR_IMAGE, while the Docker driver already resolved a version-pinned tag. Extract the tag resolution logic into openshell-core so all three drivers use the same OPENSHELL_IMAGE_TAG > IMAGE_TAG > CARGO_PKG_VERSION priority chain. Closes #2068 Signed-off-by: Florent Benoit * refactor(core): simplify supervisor image tag resolver to slice-based API Remove the Docker driver's wrapper functions and call openshell_core::config::default_supervisor_image() directly. Simplify resolve_supervisor_image_tag to accept &[&str] instead of three separate parameters. Signed-off-by: Florent Benoit Signed-off-by: Florent Benoit --------- Signed-off-by: Florent Benoit Signed-off-by: Florent Benoit --- crates/openshell-core/src/config.rs | 74 ++++++++++++++++++- crates/openshell-driver-docker/src/lib.rs | 53 +------------ crates/openshell-driver-docker/src/tests.rs | 31 +++----- .../openshell-driver-kubernetes/src/config.rs | 4 +- .../openshell-driver-kubernetes/src/main.rs | 2 +- crates/openshell-driver-podman/src/config.rs | 4 +- .../openshell-driver-podman/src/container.rs | 5 +- crates/openshell-driver-podman/src/main.rs | 6 +- docs/reference/gateway-config.mdx | 12 ++- 9 files changed, 105 insertions(+), 86 deletions(-) diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index ba6b9d401f..a0f4ec9cf5 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -37,8 +37,40 @@ pub const DEFAULT_DOCKER_NETWORK_NAME: &str = "openshell-docker"; /// Default domain used for browser-facing sandbox service URLs. pub const DEFAULT_SERVICE_ROUTING_DOMAIN: &str = "openshell.localhost"; -/// Default OCI image for the openshell-sandbox supervisor binary. -pub const DEFAULT_SUPERVISOR_IMAGE: &str = "ghcr.io/nvidia/openshell/supervisor:latest"; +/// Default OCI repository for the supervisor image (no tag). +pub const DEFAULT_SUPERVISOR_IMAGE_REPO: &str = "ghcr.io/nvidia/openshell/supervisor"; + +/// Return the default supervisor image reference with a version-pinned tag. +#[must_use] +pub fn default_supervisor_image() -> String { + format!( + "{DEFAULT_SUPERVISOR_IMAGE_REPO}:{}", + default_supervisor_image_tag() + ) +} + +fn default_supervisor_image_tag() -> String { + resolve_supervisor_image_tag(&[ + option_env!("OPENSHELL_IMAGE_TAG").unwrap_or(""), + option_env!("IMAGE_TAG").unwrap_or(""), + env!("CARGO_PKG_VERSION"), + ]) +} + +/// Resolve the supervisor image tag from an ordered list of candidates. +/// +/// Returns the first non-empty, non-`"0.0.0"` candidate, falling back to +/// `"dev"` when none qualifies. Replaces `+` with `-` for OCI tag +/// compatibility. +#[must_use] +pub fn resolve_supervisor_image_tag(candidates: &[&str]) -> String { + candidates + .iter() + .copied() + .find(|t| !t.is_empty() && *t != "0.0.0") + .unwrap_or("dev") + .replace('+', "-") +} /// CDI device identifier for requesting all NVIDIA GPUs. pub const CDI_GPU_DEVICE_ALL: &str = "nvidia.com/gpu=all"; @@ -1064,4 +1096,42 @@ mod tests { } } } + + #[test] + fn supervisor_image_tag_prefers_explicit_build_tags() { + use super::resolve_supervisor_image_tag; + assert_eq!( + resolve_supervisor_image_tag(&["1.2.3", "sha", "0.0.0"]), + "1.2.3" + ); + assert_eq!(resolve_supervisor_image_tag(&["", "sha", "0.0.0"]), "sha"); + assert_eq!(resolve_supervisor_image_tag(&["", "", "1.2.3"]), "1.2.3"); + assert_eq!(resolve_supervisor_image_tag(&["", "", "0.0.0"]), "dev"); + assert_eq!( + resolve_supervisor_image_tag(&["latest", "", "1.2.3"]), + "latest" + ); + } + + #[test] + fn supervisor_image_tag_sanitizes_build_metadata_for_oci() { + use super::resolve_supervisor_image_tag; + assert_eq!( + resolve_supervisor_image_tag(&["", "", "0.0.37-dev.156+g1d3b741ee"]), + "0.0.37-dev.156-g1d3b741ee", + ); + assert_eq!( + resolve_supervisor_image_tag(&["0.0.37-dev.156+g1d3b741ee", "", "0.0.0"]), + "0.0.37-dev.156-g1d3b741ee", + ); + } + + #[test] + fn default_supervisor_image_is_version_pinned() { + use super::default_supervisor_image; + let image = default_supervisor_image(); + assert!(image.starts_with("ghcr.io/nvidia/openshell/supervisor:")); + let tag = image.rsplit_once(':').unwrap().1; + assert!(!tag.is_empty()); + } } diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index d9bea99e7c..06d575a1e1 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -78,55 +78,6 @@ const HOST_OPENSHELL_INTERNAL: &str = "host.openshell.internal"; const HOST_DOCKER_INTERNAL: &str = "host.docker.internal"; const DOCKER_NETWORK_DRIVER: &str = "bridge"; -/// Default image holding the Linux `openshell-sandbox` binary. The gateway -/// pulls this image and extracts the binary to a host-side cache when no -/// explicit `supervisor_bin`, configured `supervisor_image`, sibling binary, -/// or local build is available. -const DEFAULT_DOCKER_SUPERVISOR_IMAGE_REPO: &str = "ghcr.io/nvidia/openshell/supervisor"; - -/// Return the default `ghcr.io/nvidia/openshell/supervisor:` reference -/// used when no supervisor binary override is provided. -pub fn default_docker_supervisor_image() -> String { - format!( - "{DEFAULT_DOCKER_SUPERVISOR_IMAGE_REPO}:{}", - default_docker_supervisor_image_tag() - ) -} - -/// Image tag baked in at compile time to pair the gateway with a matching -/// supervisor image. -/// -/// Build pipelines pass `OPENSHELL_IMAGE_TAG` explicitly. The `IMAGE_TAG` -/// fallback covers image build wrappers that already tag the gateway and -/// supervisor together. Standalone release binaries also patch the Cargo -/// package version, so use it when it has been set to a real release value. -fn default_docker_supervisor_image_tag() -> String { - resolve_default_docker_supervisor_image_tag( - option_env!("OPENSHELL_IMAGE_TAG"), - option_env!("IMAGE_TAG"), - env!("CARGO_PKG_VERSION"), - ) -} - -fn resolve_default_docker_supervisor_image_tag( - openshell_image_tag: Option<&'static str>, - image_tag: Option<&'static str>, - cargo_pkg_version: &'static str, -) -> String { - let tag = openshell_image_tag - .filter(|tag| !tag.is_empty()) - .or_else(|| image_tag.filter(|tag| !tag.is_empty())) - .unwrap_or_else(|| { - if cargo_pkg_version.is_empty() || cargo_pkg_version == "0.0.0" { - "dev" - } else { - cargo_pkg_version - } - }); - - tag.replace('+', "-") -} - /// Queried by the Docker driver to decide when a sandbox's supervisor /// relay is live. Implementations return `true` once a sandbox has an /// active `ConnectSupervisor` session registered. @@ -3079,7 +3030,9 @@ fn resolve_supervisor_bin_source( // Tier 5: pull the release-matched default supervisor image and extract // the binary to a host-side cache keyed by image content digest. - Ok(SupervisorBinSource::Image(default_docker_supervisor_image())) + Ok(SupervisorBinSource::Image( + openshell_core::config::default_supervisor_image(), + )) } pub(crate) async fn resolve_supervisor_bin( diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index d42d41099d..560d47de7a 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -2018,7 +2018,7 @@ fn docker_guest_tls_paths_allows_plain_http_without_tls_flags() { #[test] fn default_docker_supervisor_image_uses_nvidia_ghcr_repo() { - let image = default_docker_supervisor_image(); + let image = openshell_core::config::default_supervisor_image(); assert!( image.starts_with("ghcr.io/nvidia/openshell/supervisor:"), "unexpected default image reference: {image}", @@ -2057,36 +2057,25 @@ fn configured_supervisor_image_takes_precedence_over_local_binaries() { #[test] fn docker_supervisor_image_tag_prefers_explicit_build_tags() { + use openshell_core::config::resolve_supervisor_image_tag; assert_eq!( - resolve_default_docker_supervisor_image_tag(Some("1.2.3"), Some("sha"), "0.0.0"), - "1.2.3", - ); - assert_eq!( - resolve_default_docker_supervisor_image_tag(None, Some("sha"), "0.0.0"), - "sha", - ); - assert_eq!( - resolve_default_docker_supervisor_image_tag(None, None, "1.2.3"), - "1.2.3", - ); - assert_eq!( - resolve_default_docker_supervisor_image_tag(Some(""), Some(""), "0.0.0"), - "dev", + resolve_supervisor_image_tag(&["1.2.3", "sha", "0.0.0"]), + "1.2.3" ); + assert_eq!(resolve_supervisor_image_tag(&["", "sha", "0.0.0"]), "sha"); + assert_eq!(resolve_supervisor_image_tag(&["", "", "1.2.3"]), "1.2.3"); + assert_eq!(resolve_supervisor_image_tag(&["", "", "0.0.0"]), "dev"); } #[test] fn docker_supervisor_image_tag_sanitizes_build_metadata_for_docker() { + use openshell_core::config::resolve_supervisor_image_tag; assert_eq!( - resolve_default_docker_supervisor_image_tag(None, None, "0.0.37-dev.156+g1d3b741ee"), + resolve_supervisor_image_tag(&["", "", "0.0.37-dev.156+g1d3b741ee"]), "0.0.37-dev.156-g1d3b741ee", ); assert_eq!( - resolve_default_docker_supervisor_image_tag( - Some("0.0.37-dev.156+g1d3b741ee"), - None, - "0.0.0", - ), + resolve_supervisor_image_tag(&["0.0.37-dev.156+g1d3b741ee", "", "0.0.0"]), "0.0.37-dev.156-g1d3b741ee", ); } diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 292563c2e6..23f80f680b 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use openshell_core::config::DEFAULT_SUPERVISOR_IMAGE; +use openshell_core::config; use serde::{Deserialize, Deserializer, Serialize}; use std::path::Path; use std::str::FromStr; @@ -288,7 +288,7 @@ impl Default for KubernetesComputeConfig { // is Podman vocabulary and is not a valid Kubernetes value. image_pull_policy: String::new(), image_pull_secrets: Vec::new(), - supervisor_image: DEFAULT_SUPERVISOR_IMAGE.to_string(), + supervisor_image: config::default_supervisor_image(), supervisor_image_pull_policy: String::new(), supervisor_sideload_method: SupervisorSideloadMethod::default(), supervisor_topology: SupervisorTopology::default(), diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 77f671dcb2..a300fe6a07 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -127,7 +127,7 @@ async fn main() -> Result<()> { image_pull_secrets: args.sandbox_image_pull_secrets, supervisor_image: args .supervisor_image - .unwrap_or_else(|| openshell_core::config::DEFAULT_SUPERVISOR_IMAGE.to_string()), + .unwrap_or_else(openshell_core::config::default_supervisor_image), supervisor_image_pull_policy: args.supervisor_image_pull_policy.unwrap_or_default(), supervisor_sideload_method: args.supervisor_sideload_method, supervisor_topology: args.supervisor_topology, diff --git a/crates/openshell-driver-podman/src/config.rs b/crates/openshell-driver-podman/src/config.rs index 0e29f52dd2..def8e5f3dc 100644 --- a/crates/openshell-driver-podman/src/config.rs +++ b/crates/openshell-driver-podman/src/config.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use openshell_core::config::{DEFAULT_STOP_TIMEOUT_SECS, DEFAULT_SUPERVISOR_IMAGE}; +use openshell_core::config::DEFAULT_STOP_TIMEOUT_SECS; use std::net::IpAddr; use std::path::PathBuf; use std::str::FromStr; @@ -255,7 +255,7 @@ impl Default for PodmanComputeConfig { network_name: DEFAULT_NETWORK_NAME.to_string(), host_gateway_ip: Self::default_host_gateway_ip(), stop_timeout_secs: DEFAULT_STOP_TIMEOUT_SECS, - supervisor_image: DEFAULT_SUPERVISOR_IMAGE.to_string(), + supervisor_image: openshell_core::config::default_supervisor_image(), guest_tls_ca: None, guest_tls_cert: None, guest_tls_key: None, diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index ad44fca43e..5eb9a5c873 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -1836,7 +1836,7 @@ mod tests { let vol = &image_volumes[0]; assert_eq!( vol["source"].as_str(), - Some("ghcr.io/nvidia/openshell/supervisor:latest"), + Some(openshell_core::config::default_supervisor_image().as_str()), "image volume source should be the supervisor image" ); assert_eq!( @@ -1922,8 +1922,9 @@ mod tests { let image_volumes = spec["image_volumes"] .as_array() .expect("image_volumes should be an array"); + let expected_supervisor = openshell_core::config::default_supervisor_image(); assert!(image_volumes.iter().any(|volume| { - volume["source"].as_str() == Some("ghcr.io/nvidia/openshell/supervisor:latest") + volume["source"].as_str() == Some(expected_supervisor.as_str()) && volume["destination"].as_str() == Some("/opt/openshell/bin") })); assert!(image_volumes.iter().any(|volume| { diff --git a/crates/openshell-driver-podman/src/main.rs b/crates/openshell-driver-podman/src/main.rs index e6ba7b9ffc..c57aff4277 100644 --- a/crates/openshell-driver-podman/src/main.rs +++ b/crates/openshell-driver-podman/src/main.rs @@ -90,7 +90,7 @@ struct Args { /// OCI image containing the openshell-sandbox supervisor binary. #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE")] - supervisor_image: String, + supervisor_image: Option, /// Host path to the CA certificate for sandbox mTLS. #[arg(long, env = "OPENSHELL_PODMAN_TLS_CA")] @@ -130,7 +130,9 @@ async fn main() -> Result<()> { sandbox_ssh_socket_path: args.sandbox_ssh_socket_path, network_name: args.network_name, stop_timeout_secs: args.stop_timeout, - supervisor_image: args.supervisor_image, + supervisor_image: args + .supervisor_image + .unwrap_or_else(openshell_core::config::default_supervisor_image), guest_tls_ca: args.podman_tls_ca, guest_tls_cert: args.podman_tls_cert, guest_tls_key: args.podman_tls_key, diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 88b82870de..8f840d1776 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -88,7 +88,8 @@ disable_tls = false # Shared driver defaults. These inherit into [openshell.drivers.] tables # when the driver-specific table does not override them. default_image = "ghcr.io/nvidia/openshell/sandbox:latest" -supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" +# Defaults to the gateway version; override to pin a specific build. +# supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" client_tls_secret_name = "openshell-client-tls" service_account_name = "openshell-sandbox" host_gateway_ip = "10.0.0.1" @@ -172,7 +173,8 @@ service_account_name = "openshell-sandbox" default_image = "ghcr.io/nvidia/openshell/sandbox:latest" image_pull_policy = "IfNotPresent" image_pull_secrets = ["regcred"] -supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" +# Defaults to the gateway version; override to pin a specific build. +# supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" supervisor_image_pull_policy = "IfNotPresent" # Use the image volume on Kubernetes >= 1.35 (GA in 1.36); switch to "init-container" # on older clusters or where the ImageVolume feature gate is off. @@ -228,7 +230,8 @@ grpc_endpoint = "https://host.openshell.internal:17670" # Skip the image-pull-and-extract step by pointing at a locally built binary. supervisor_bin = "/usr/local/libexec/openshell/openshell-sandbox" # When supervisor_bin is omitted, Docker extracts /openshell-sandbox from this image. -supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" +# Defaults to the gateway version; override to pin a specific build. +# supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" guest_tls_ca = "/etc/openshell/certs/ca.pem" guest_tls_cert = "/etc/openshell/certs/client.pem" guest_tls_key = "/etc/openshell/certs/client-key.pem" @@ -270,7 +273,8 @@ network_name = "openshell" # host_gateway_ip = "192.168.127.254" sandbox_ssh_socket_path = "/run/openshell/ssh.sock" stop_timeout_secs = 10 -supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" +# Defaults to the gateway version; override to pin a specific build. +# supervisor_image = "ghcr.io/nvidia/openshell/supervisor:" guest_tls_ca = "/etc/openshell/certs/ca.pem" guest_tls_cert = "/etc/openshell/certs/client.pem" guest_tls_key = "/etc/openshell/certs/client-key.pem" From bebf440b2545f68a59e70848b6d326b1ec9bc972 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Fri, 10 Jul 2026 12:31:11 -0700 Subject: [PATCH 07/13] fix(helm): propagate supervisor image overrides (#2216) Signed-off-by: Taylor Mutch --- architecture/build.md | 3 ++ deploy/helm/openshell/README.md | 4 +-- deploy/helm/openshell/templates/_helpers.tpl | 23 ++++++++++-- .../openshell/templates/gateway-config.yaml | 2 ++ .../openshell/tests/gateway_config_test.yaml | 36 +++++++++++++++++++ deploy/helm/openshell/values.yaml | 10 +++--- docs/reference/sandbox-compute-drivers.mdx | 2 +- 7 files changed, 69 insertions(+), 11 deletions(-) diff --git a/architecture/build.md b/architecture/build.md index a3cb2e25fb..5537e63619 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -98,6 +98,9 @@ Runtime layout: Gateway image builds bake the corresponding supervisor image tag into the gateway binary so Docker sandboxes do not depend on `:latest` by default. +The Helm chart omits the supervisor image from gateway configuration unless an +operator supplies a repository or tag override, preserving that build-time +pairing for Kubernetes sandboxes as well. Package formulas also pin Docker supervisor extraction to the matching release image tag so standalone gateway binaries do not infer image tags from package versions. diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 6bc459567b..4a22d640f5 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -237,8 +237,8 @@ add `ci/values-spire.yaml` to the OpenShell release values files. | serviceAccount.create | bool | `true` | Create a service account for the gateway. | | serviceAccount.name | string | `""` | Existing service account name to use when serviceAccount.create is false. | | supervisor.image.pullPolicy | string | `""` | Supervisor image pull policy. Defaults to the gateway image pull policy when empty. | -| supervisor.image.repository | string | `"ghcr.io/nvidia/openshell/supervisor"` | Supervisor image repository. | -| supervisor.image.tag | string | `""` | Supervisor image tag. Defaults to the chart appVersion when empty. | +| supervisor.image.repository | string | `"ghcr.io/nvidia/openshell/supervisor"` | Supervisor image repository. Changing it uses the effective gateway image tag unless tag is also set. | +| supervisor.image.tag | string | `""` | Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed. | | supervisor.sideloadMethod | string | `""` | How the supervisor binary is delivered into sandbox pods. Empty (default) = auto-detect from cluster version: K8s >= v1.35 -> "image-volume" (ImageVolume enabled by default; GA in v1.36) K8s < v1.35 -> "init-container" (copies via init container + emptyDir) On K8s v1.33-v1.34 with the ImageVolume feature gate manually enabled, set this to "image-volume" explicitly. | | supervisor.topology | string | `"combined"` | Supervisor pod topology for Kubernetes sandboxes. "combined" runs networking and process supervision in the agent container. | | tolerations | list | `[]` | Tolerations for the gateway pod. | diff --git a/deploy/helm/openshell/templates/_helpers.tpl b/deploy/helm/openshell/templates/_helpers.tpl index 5872dc4043..1b4598088f 100644 --- a/deploy/helm/openshell/templates/_helpers.tpl +++ b/deploy/helm/openshell/templates/_helpers.tpl @@ -78,12 +78,29 @@ so a released chart automatically pulls the matching image without extra overrid {{- printf "%s:%s" .Values.image.repository (.Values.image.tag | default .Chart.AppVersion) }} {{- end }} +{{/* Official supervisor repository used by the gateway's built-in default. */}} +{{- define "openshell.defaultSupervisorRepository" -}} +ghcr.io/nvidia/openshell/supervisor +{{- end }} + +{{/* +Whether Helm must propagate a supervisor image override into gateway.toml. +The chart's documented repository and empty tag are the gateway-owned default. +*/}} +{{- define "openshell.supervisorImageOverrideEnabled" -}} +{{- $defaultRepository := include "openshell.defaultSupervisorRepository" . -}} +{{- $repository := .Values.supervisor.image.repository | default $defaultRepository -}} +{{- if or (ne $repository $defaultRepository) .Values.supervisor.image.tag -}}true{{- end -}} +{{- end }} + {{/* -Supervisor image reference. Same appVersion fallback as openshell.image so -the supervisor and gateway images stay in sync across releases. +Supervisor image override. A tag-only override uses the official repository; +a repository-only override uses the effective gateway image tag. */}} {{- define "openshell.supervisorImage" -}} -{{- printf "%s:%s" .Values.supervisor.image.repository (.Values.supervisor.image.tag | default .Chart.AppVersion) }} +{{- $repository := .Values.supervisor.image.repository | default (include "openshell.defaultSupervisorRepository" .) -}} +{{- $tag := .Values.supervisor.image.tag | default .Values.image.tag | default .Chart.AppVersion -}} +{{- printf "%s:%s" $repository $tag }} {{- end }} {{/* diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 0e75a311f0..55beac7a19 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -34,7 +34,9 @@ data: log_level = {{ .Values.server.logLevel | quote }} sandbox_namespace = {{ include "openshell.sandboxNamespace" . | quote }} default_image = {{ .Values.server.sandboxImage | quote }} + {{- if include "openshell.supervisorImageOverrideEnabled" . }} supervisor_image = {{ include "openshell.supervisorImage" . | quote }} + {{- end }} {{- if .Values.server.hostGatewayIP }} host_gateway_ip = {{ .Values.server.hostGatewayIP | quote }} {{- end }} diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index 50051e0036..797182558e 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -90,6 +90,42 @@ tests: path: data["gateway.toml"] pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?supervisor_topology\s*=\s*"combined"' + - it: uses the gateway built-in supervisor image by default + template: templates/gateway-config.yaml + asserts: + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'supervisor_image\s*=' + + - it: renders a supervisor tag override with the official repository + template: templates/gateway-config.yaml + set: + supervisor.image.tag: 1.2.3 + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: 'supervisor_image\s*=\s*"ghcr\.io/nvidia/openshell/supervisor:1\.2\.3"' + + - it: renders a supervisor repository override with the effective gateway tag + template: templates/gateway-config.yaml + set: + image.tag: gateway-build + supervisor.image.repository: registry.example.com/openshell/supervisor + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: 'supervisor_image\s*=\s*"registry\.example\.com/openshell/supervisor:gateway-build"' + + - it: renders complete supervisor repository and tag overrides + template: templates/gateway-config.yaml + set: + supervisor.image.repository: registry.example.com/openshell/supervisor + supervisor.image.tag: supervisor-build + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: 'supervisor_image\s*=\s*"registry\.example\.com/openshell/supervisor:supervisor-build"' + - it: renders explicit combined supervisor topology under [openshell.drivers.kubernetes] template: templates/gateway-config.yaml set: diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index fd73dd0713..f36b533d13 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -26,16 +26,16 @@ image: # -- Gateway image tag. Defaults to the chart appVersion when empty. tag: "" -# Supervisor image - provides the openshell-sandbox binary injected into sandbox -# pods. tag defaults to appVersion (same as the gateway image) so both stay in -# sync when the chart is released. +# Supervisor image for the openshell-sandbox binary injected into sandbox pods. +# The default repository and empty tag use the version-pinned image built into +# the gateway. Changing the repository or setting a tag enables a Helm override. supervisor: image: - # -- Supervisor image repository. + # -- Supervisor image repository. Changing it uses the effective gateway image tag unless tag is also set. repository: ghcr.io/nvidia/openshell/supervisor # -- Supervisor image pull policy. Defaults to the gateway image pull policy when empty. pullPolicy: "" - # -- Supervisor image tag. Defaults to the chart appVersion when empty. + # -- Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed. tag: "" # -- How the supervisor binary is delivered into sandbox pods. # Empty (default) = auto-detect from cluster version: diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index 714ddd6c33..e5cb8e254d 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -303,7 +303,7 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA | `image_pull_secrets` | `server.sandboxImagePullSecrets` | Attach Kubernetes image pull secrets to sandbox pods. Referenced Secrets must exist in the sandbox namespace. | | `grpc_endpoint` | `server.grpcEndpoint` | Set the gateway callback endpoint reachable from sandbox pods. | | `client_tls_secret_name` | `server.tls.clientTlsSecretName` | Mount sandbox client TLS materials from a Kubernetes secret. | -| `supervisor_image` | `supervisor.image.repository` / `supervisor.image.tag` | Set the supervisor image that provides the `openshell-sandbox` binary. | +| `supervisor_image` | `supervisor.image.repository` / `supervisor.image.tag` | Override the supervisor image that provides the `openshell-sandbox` binary. The default repository with an empty tag uses the version-pinned image built into the gateway. Changing the repository uses the effective gateway image tag, while setting a tag pins that version explicitly. | | `supervisor_image_pull_policy` | `supervisor.image.pullPolicy` | Set the Kubernetes image pull policy for the supervisor image. | | `supervisor_sideload_method` | `supervisor.sideloadMethod` | How the supervisor binary is delivered into sandbox pods. Leave empty to auto-detect from cluster version. Set to `image-volume` to mount the supervisor OCI image directly as a volume (requires Kubernetes 1.33+ with the ImageVolume feature gate; GA in 1.36), or `init-container` to copy it through an init container on older clusters. | | `app_armor_profile` | `server.appArmorProfile` | Set the sandbox agent container's AppArmor profile. Helm defaults this to `Unconfined` so AppArmor-enabled nodes do not block supervisor network namespace setup. Set the Helm value to an empty string to omit the field, or use `RuntimeDefault` or `Localhost/` for operator-managed profiles. | From 8eacb4779f7fe5fc56dfe2834421f78a480e5474 Mon Sep 17 00:00:00 2001 From: Taylor Mutch Date: Fri, 10 Jul 2026 13:01:39 -0700 Subject: [PATCH 08/13] feat(kubernetes): add sidecar supervisor topology (#2076) * feat(kubernetes): add sidecar supervisor topology Add the Kubernetes sidecar supervisor topology, its Helm/Skaffold configuration, topology documentation, and sidecar e2e matrix coverage. Skip root-only sandbox identity rewriting when process enforcement is network-only so the low-permission sidecar process container can start successfully. Signed-off-by: Taylor Mutch * fix(supervisor): avoid similar process id names Signed-off-by: Taylor Mutch * fix(supervisor): avoid similar process id names Signed-off-by: Taylor Mutch * fix(sandbox): avoid similar proxy id names Signed-off-by: Taylor Mutch * docs(kubernetes): clarify sidecar topology limits Signed-off-by: Taylor Mutch * fix(kubernetes): keep sidecar process leaf capless Signed-off-by: Taylor Mutch * fix(kubernetes): refresh sidecar provider env snapshots Signed-off-by: Taylor Mutch * test(supervisor): align hot-swap identity regression Signed-off-by: Taylor Mutch * fix(kubernetes): stage sidecar mtls files before proxy chown Signed-off-by: Taylor Mutch * fix(kubernetes): simplify sidecar supervisor topology Signed-off-by: Taylor Mutch * chore(helm): reuse sidecar skaffold values Signed-off-by: Taylor Mutch * fix(supervisor): avoid similar iptables helper names Signed-off-by: Taylor Mutch * fix(e2e): harden kube gateway wrapper setup Signed-off-by: Taylor Mutch * fix(supervisor): avoid nft batch rollback on OCP Run nftables setup as individual commands so optional conntrack and log expressions can fail without rolling back required table, chain, and reject rules. Signed-off-by: Seth Jennings Signed-off-by: Taylor Mutch * fix(kubernetes): preserve process identity in sidecar topology Render sidecar pods with a shared process namespace, keep binary-aware network policy enabled, and move Kubernetes sidecar settings under the nested sidecar config table. Also apply unprivileged Landlock/seccomp setup in NetworkOnly supervisor mode so sidecar topology keeps sandbox child hardening without privileged process setup. Signed-off-by: Taylor Mutch * refactor(kubernetes): replace sidecar snapshots with control socket Coordinate sidecar policy and provider bootstrap over a local Unix socket so the process leaf no longer reads policy/provider snapshot files. Report entrypoint startup through the control channel and keep gateway credentials confined to the network sidecar. Signed-off-by: Taylor Mutch * feat(kubernetes): support relaxed sidecar network identity Signed-off-by: Taylor Mutch * fix(sandbox): satisfy sidecar clippy lint Signed-off-by: Taylor Mutch * refactor(kubernetes): standardize topology naming Signed-off-by: Taylor Mutch * fix(sandbox): satisfy linux clippy timeout import Signed-off-by: Taylor Mutch * fix(kubernetes): support kata sidecar on ipv4 pods Signed-off-by: Taylor Mutch * fix(kubernetes): satisfy linux clippy for sidecar fallback Signed-off-by: Taylor Mutch * chore(kubernetes): remove stale supervisor topology references Signed-off-by: Taylor Mutch * fix(kubernetes): enable sidecar binary policy inspection Signed-off-by: Taylor Mutch * fix(kubernetes): harden sidecar control boundary Signed-off-by: Taylor Mutch * fix(kubernetes): couple sidecar supervisor lifecycles Signed-off-by: Taylor Mutch --------- Signed-off-by: Taylor Mutch Signed-off-by: Seth Jennings Co-authored-by: Seth Jennings --- .../skills/debug-openshell-cluster/SKILL.md | 51 + .agents/skills/helm-dev-environment/SKILL.md | 30 +- .github/workflows/branch-e2e.yml | 11 +- Cargo.lock | 3 + architecture/build.md | 9 +- architecture/compute-runtimes.md | 40 +- crates/openshell-core/src/grpc_client.rs | 7 +- .../src/provider_credentials.rs | 90 ++ crates/openshell-core/src/sandbox_env.rs | 28 + crates/openshell-driver-kubernetes/README.md | 40 +- .../openshell-driver-kubernetes/src/config.rs | 186 +++- .../openshell-driver-kubernetes/src/driver.rs | 981 +++++++++++++++++- crates/openshell-driver-kubernetes/src/lib.rs | 5 +- .../openshell-driver-kubernetes/src/main.rs | 33 +- crates/openshell-driver-podman/README.md | 4 +- .../openshell-driver-podman/src/container.rs | 5 +- crates/openshell-sandbox/Cargo.toml | 5 + crates/openshell-sandbox/src/lib.rs | 715 +++++++++++-- crates/openshell-sandbox/src/main.rs | 299 +++++- .../openshell-sandbox/src/sidecar_control.rs | 783 ++++++++++++++ .../data/sandbox-policy.rego | 14 + .../src/identity.rs | 69 +- .../openshell-supervisor-network/src/opa.rs | 202 +++- .../openshell-supervisor-network/src/proxy.rs | 322 ++++-- .../openshell-supervisor-network/src/run.rs | 4 +- .../openshell-supervisor-process/src/lib.rs | 2 + .../src/netns/mod.rs | 419 ++++++-- .../src/netns/nft_ruleset.rs | 525 ++++++++-- .../src/process.rs | 109 +- .../openshell-supervisor-process/src/run.rs | 124 ++- .../src/sandbox/linux/landlock.rs | 78 +- .../src/sandbox/linux/mod.rs | 17 +- .../openshell-supervisor-process/src/ssh.rs | 215 +++- .../src/supervisor_session.rs | 84 +- .../src/unix_socket.rs | 60 ++ deploy/docker/Dockerfile.supervisor | 25 +- deploy/helm/openshell/README.md | 4 +- .../openshell/ci/values-sidecar-kata.yaml | 24 + deploy/helm/openshell/ci/values-sidecar.yaml | 18 + deploy/helm/openshell/skaffold.yaml | 17 + .../openshell/templates/gateway-config.yaml | 6 +- .../openshell/tests/gateway_config_test.yaml | 32 +- deploy/helm/openshell/values.yaml | 16 +- docs/kubernetes/access-control.mdx | 2 +- docs/kubernetes/ingress.mdx | 2 +- docs/kubernetes/managing-certificates.mdx | 2 +- docs/kubernetes/openshift.mdx | 7 +- docs/kubernetes/setup.mdx | 3 +- docs/kubernetes/topology.mdx | 195 +++- docs/reference/gateway-config.mdx | 19 +- docs/reference/sandbox-compute-drivers.mdx | 28 + e2e/with-kube-gateway.sh | 19 +- tasks/helm.toml | 30 + tasks/test.toml | 5 + 54 files changed, 5463 insertions(+), 560 deletions(-) create mode 100644 crates/openshell-sandbox/src/sidecar_control.rs create mode 100644 crates/openshell-supervisor-process/src/unix_socket.rs create mode 100644 deploy/helm/openshell/ci/values-sidecar-kata.yaml create mode 100644 deploy/helm/openshell/ci/values-sidecar.yaml diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index 5bc04beb39..82ce9650cd 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -268,6 +268,57 @@ kubectl -n openshell get configmap openshell-config -o jsonpath='{.data.gateway\ kubectl -n get sandbox -o jsonpath='{.spec.template.spec.serviceAccountName}{"\n"}' ``` +If `topology = "sidecar"` is rendered under `[openshell.drivers.kubernetes]`, +sandbox pods should have an `openshell-network-init` init container running +`--mode=network-init`, an `agent` container running +`openshell-sandbox --mode=process`, and an `openshell-supervisor-network` +container running `--mode=network`. The init container owns nftables setup and +should be the only sidecar topology container with `NET_ADMIN`. It also needs +`CHOWN`/`FOWNER` to hand shared emptyDir state to the effective sidecar UID. The +default binary-aware network sidecar runs as UID 0 with primary GID +`sandbox_gid` and adds `SYS_PTRACE` plus `DAC_READ_SEARCH`. When +`process_binary_aware_network_policy = false`, it runs as the configured +non-root `proxy_uid` without those inspection capabilities. The pod `fsGroup` +is set to `sandbox_gid` in both modes. + +In sidecar topology only the network sidecar should mount the gateway bootstrap +credentials (`openshell-sa-token` and `openshell-client-tls`). The process +container should not receive `OPENSHELL_ENDPOINT`, gateway TLS env vars, the +sandbox token file, or those credential mounts. Instead, the network sidecar +serves policy and provider environment state over the Unix control socket from +`OPENSHELL_SIDECAR_CONTROL_SOCKET` (`/run/openshell-sidecar/control.sock` by +default). The process supervisor must be the first and only client. After +validating its peer UID, GID, and PID, the sidecar unlinks the listener. If the +connection later closes, the network sidecar exits non-zero so Kubernetes can +restart it with a fresh listener. If the process supervisor fails before +launching the workload, +inspect both containers for control-socket bind, connect, bootstrap, or update +errors. If new SSH/exec sessions do not pick up refreshed provider environment, +inspect the network sidecar settings-poll logs and the process container logs +for provider environment update handling; the process container should consume +newer provider-env revisions without receiving gateway credentials. + +The process container reports the workload entrypoint PID over the same control +socket, and the network sidecar uses that PID for binary-scoped policy +decisions through `/proc`. If rules with `policy.binaries` are unexpectedly +denied, inspect the sidecar control logs and confirm the pod has +`shareProcessNamespace: true`. +The shared state directory should preserve `sandbox_gid` inheritance +(`02775`). Sidecar SSH uses the Linux abstract socket +`@openshell-sidecar-ssh`; the network sidecar verifies its peer PID before +bridging gateway relay requests. No `ssh.sock` file should appear in the shared +state directory. +Inspect all three when sandbox registration or egress enforcement fails: + +```bash +kubectl -n openshell get configmap openshell-config -o jsonpath='{.data.gateway\.toml}' | grep -E '^\[openshell\.drivers\.kubernetes\]|^topology\s*=' +kubectl -n get pod -o jsonpath='{range .spec.initContainers[*]}{.name}{" "}{.command}{"\n"}{end}' +kubectl -n get pod -o jsonpath='{range .spec.containers[*]}{.name}{" "}{.command}{"\n"}{end}' +kubectl -n logs -c openshell-network-init --tail=200 +kubectl -n logs -c openshell-supervisor-network --tail=200 +kubectl -n logs -c agent --tail=200 +``` + ### Step 6: Check VM-Backed Gateways Use the VM driver logs and host diagnostics available in the user's environment. Verify: diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index bffa4e2e86..adb40a9575 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -60,9 +60,25 @@ mise run helm:skaffold:dev mise run helm:skaffold:run ``` +**Supervisor sidecar topology** (build once and leave running): +```bash +mise run helm:skaffold:run:sidecar +``` + +**Supervisor sidecar topology with TLS/mTLS enabled** (build once and leave running): +```bash +mise run helm:skaffold:run:sidecar-mtls +``` + Both commands build the `gateway` and `supervisor` images and deploy the OpenShell Helm -chart. The `pkiInitJob` hook (a pre-install Job that runs `openshell-gateway generate-certs`) -generates mTLS secrets on first install. Envoy Gateway opt-in; see the Optional Add-ons section below. +chart. The sidecar profile renders an `openshell-network-init` init container for +nftables setup and an `openshell-supervisor-network` runtime sidecar for proxying. +Binary-aware policy mode runs that sidecar as UID 0 with `SYS_PTRACE` and +`DAC_READ_SEARCH`; relaxed mode can run it as the configured proxy UID. The +sidecar-mTLS profile reuses `ci/values-sidecar.yaml` and restores +`server.disableTls=false` inline for Skaffold. The `pkiInitJob` hook (a pre-install +Job that runs `openshell-gateway generate-certs`) generates mTLS secrets on first +install. Envoy Gateway opt-in; see the Optional Add-ons section below. The gateway Service uses ClusterIP. Access is via Envoy Gateway (port `8080`) or `kubectl port-forward`. @@ -74,7 +90,8 @@ create the Secret named `openshell-ha-pg` with a `uri` key, then run ### TLS behaviour `ci/values-skaffold.yaml` sets `server.disableTls: true`, so Skaffold-based deploys run -plaintext by default. To test with TLS enabled, comment out that line and redeploy. +plaintext by default. To test sidecar topology with TLS enabled, use +`mise run helm:skaffold:run:sidecar-mtls`. | Mode | `server.disableTls` | Gateway scheme | |------|---------------------|----------------| @@ -126,6 +143,12 @@ openshell sandbox list --gateway-endpoint https://localhost:8090 mise run helm:skaffold:delete ``` +For a sidecar-profile deployment: + +```bash +mise run helm:skaffold:delete:sidecar +``` + ### Delete the cluster entirely ```bash @@ -250,6 +273,7 @@ for dependencies still declared in `Chart.yaml`. | `deploy/helm/openshell/ci/values-gateway.yaml` | Envoy Gateway GRPCRoute + Gateway overlay | | `deploy/helm/openshell/ci/values-high-availability.yaml` | HA test overlay (`replicaCount: 2` with external PostgreSQL Secret) | | `deploy/helm/openshell/ci/values-keycloak.yaml` | Keycloak OIDC overlay | +| `deploy/helm/openshell/ci/values-sidecar.yaml` | Supervisor sidecar topology overlay for Kubernetes e2e/dev | | `deploy/helm/openshell/ci/values-spire.yaml` | SPIFFE/SPIRE provider token grant overlay | | `deploy/helm/openshell/ci/values-spire-stack.yaml` | SPIRE hardened chart values for local dev | | `deploy/helm/openshell/ci/values-tls-disabled.yaml` | Lint-only: TLS + auth disabled (reverse-proxy edge termination) | diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index ebe7834068..859f22ef95 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -121,16 +121,25 @@ jobs: include: - agent_sandbox_api: v1beta1 agent_sandbox_version: v0.5.0 + topology: combined + extra_helm_values: "" - agent_sandbox_api: v1alpha1 agent_sandbox_version: v0.4.6 + topology: combined + extra_helm_values: "" + - agent_sandbox_api: v1beta1 + agent_sandbox_version: v0.5.0 + topology: sidecar + extra_helm_values: deploy/helm/openshell/ci/values-sidecar.yaml permissions: contents: read packages: read uses: ./.github/workflows/e2e-kubernetes-test.yml with: image-tag: ${{ github.sha }} - job-name: Kubernetes E2E (Rust smoke, Agent Sandbox ${{ matrix.agent_sandbox_api }}) + job-name: Kubernetes E2E (Rust smoke, ${{ matrix.topology }}, Agent Sandbox ${{ matrix.agent_sandbox_api }}) agent-sandbox-version: ${{ matrix.agent_sandbox_version }} + extra-helm-values: ${{ matrix.extra_helm_values }} kubernetes-ha-e2e: needs: [pr_metadata, build-gateway, build-supervisor] diff --git a/Cargo.lock b/Cargo.lock index 916969ac5f..76dedf0625 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3827,12 +3827,15 @@ dependencies = [ "clap", "futures", "miette", + "nix", "openshell-core", "openshell-ocsf", "openshell-policy", "openshell-supervisor-network", "openshell-supervisor-process", + "prost", "rustls", + "serde", "serde_json", "temp-env", "tempfile", diff --git a/architecture/build.md b/architecture/build.md index 5537e63619..ef62a6c30c 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -91,10 +91,11 @@ Runtime layout: as a release artifact. Linux GNU VM driver binaries must not reference `GLIBC_*` symbols newer than `GLIBC_2.28`; release workflows verify this before publishing artifacts. -- **Supervisor**: `scratch` base, static musl binary at `/openshell-sandbox`. - Static linkage is required because the image is mounted/extracted into - sandbox environments (Docker extraction, Podman image volumes, Kubernetes - init-container copy-self) and cannot rely on a dynamic loader. +- **Supervisor**: Alpine base with `nftables`, static musl binary at + `/openshell-sandbox`. Static linkage keeps the binary usable when the image + is mounted/extracted into sandbox environments (Docker extraction, Podman + image volumes, Kubernetes init-container copy-self), while `nftables` supports + Kubernetes supervisor sidecar egress enforcement. Gateway image builds bake the corresponding supervisor image tag into the gateway binary so Docker sandboxes do not depend on `:latest` by default. diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index f122fda5d7..d5d0912e23 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -81,7 +81,7 @@ The supervisor must be available inside each sandbox workload: |---|---| | Docker | Bind-mounted local supervisor binary, or a binary extracted from the configured supervisor image. | | Podman | Read-only OCI image volume containing the supervisor binary. | -| Kubernetes | Sandbox pod image or pod template configuration. | +| Kubernetes | Supervisor image side-loaded into the sandbox pod by image volume or init container. | | VM | Embedded in the guest rootfs bundle. | | Extension | Defined by the out-of-tree driver. | @@ -89,6 +89,44 @@ Driver-controlled environment variables must override sandbox image or template values for sandbox ID, sandbox name, gateway endpoint, relay socket path, TLS paths, and command metadata. +Kubernetes can run the supervisor in the default combined topology or in a +sidecar topology. Combined mode keeps network and process supervision in the +agent container. Sidecar mode runs network enforcement, the proxy, and gateway +session in a dedicated sidecar, while the agent container runs only the +process-supervision leaf and launches the user workload after the sidecar +serves bootstrap state over a local control socket. The network sidecar owns +gateway credentials and sends policy plus workload-facing provider environment +state to the process leaf over that socket. It also streams provider +environment updates after settings polls so future process sessions see +updated provider env without giving the process leaf gateway access. The +pre-workload process supervisor is the only accepted control client: the +network sidecar verifies its UID, GID, and PID with peer credentials, removes +the listener after accepting it, and ignores workload-supplied relay targets. +SSH relays use a Linux abstract socket and verify its peer PID against that +authenticated process-supervisor connection, so workload filesystem access +cannot replace the relay endpoint. Either supervisor exits when this control +connection closes. This couples their restart lifecycle and prevents a workload +that survives an isolated network-sidecar restart from becoming the next +authoritative control client. In sidecar mode, an init container performs the +privileged pod-network nftables setup with +`NET_ADMIN`. The default binary-aware network sidecar runs as UID 0 without +`NET_ADMIN` and adds `SYS_PTRACE` plus `DAC_READ_SEARCH` so it can resolve +cross-UID workload process/binary identity through shared `/proc`. Operators +can set the sidecar `process_binary_aware_network_policy` flag false to run the +sidecar as the configured non-root proxy UID, omit both inspection capabilities, +and downgrade network policy to endpoint/L7 matching without `policy.binaries`. +The init path applies nftables as individual commands so optional conntrack and +log expressions can fail without rolling back the required table, chain, and +reject rules. +The agent container runs as the resolved sandbox UID/GID with no added Linux +capabilities. Sidecar mode preserves gateway session and SSH behavior, but +treats the process leaf as network-only: Landlock filesystem policy and child +seccomp still apply where supported, while process privilege dropping and +supervisor identity mount isolation do not run because the agent container is +already unprivileged. Sidecar pods use a shared process namespace so the +network sidecar can resolve workload process and binary identity through +`/proc/`. + ## Images The gateway image and Helm chart are built from this repository. Sandbox images diff --git a/crates/openshell-core/src/grpc_client.rs b/crates/openshell-core/src/grpc_client.rs index 57f6bdd816..92eab00408 100644 --- a/crates/openshell-core/src/grpc_client.rs +++ b/crates/openshell-core/src/grpc_client.rs @@ -167,9 +167,14 @@ async fn build_plain_channel(endpoint: &str) -> Result { .into_diagnostic() .wrap_err_with(|| format!("failed to read client key from {key_path}"))?; - let tls_config = ClientTlsConfig::new() + let mut tls_config = ClientTlsConfig::new() .ca_certificate(Certificate::from_pem(ca_pem)) .identity(Identity::from_pem(cert_pem, key_pem)); + if let Ok(server_name) = std::env::var(sandbox_env::GATEWAY_TLS_SERVER_NAME) + && !server_name.is_empty() + { + tls_config = tls_config.domain_name(server_name); + } ep = ep .tls_config(tls_config) diff --git a/crates/openshell-core/src/provider_credentials.rs b/crates/openshell-core/src/provider_credentials.rs index fd38453ccd..65310b3a43 100644 --- a/crates/openshell-core/src/provider_credentials.rs +++ b/crates/openshell-core/src/provider_credentials.rs @@ -63,6 +63,63 @@ impl ProviderCredentialState { } } + /// Build a static provider state from an already-prepared child + /// environment snapshot. + /// + /// Kubernetes sidecar topology uses this in the process-only supervisor: + /// the network sidecar owns provider credential resolvers and sends the + /// workload-facing env map over a local control channel. The process leaf + /// must inject that map into child processes without re-placeholderizing it + /// or holding the gateway-side resolver material. + pub fn from_child_env_snapshot(revision: u64, child_env: HashMap) -> Self { + let snapshot = Arc::new(ProviderCredentialSnapshot { + revision, + child_env, + dynamic_credentials: HashMap::new(), + }); + + Self { + inner: Arc::new(RwLock::new(ProviderCredentialStateInner { + current: snapshot, + generations: VecDeque::new(), + current_resolver: None, + combined_resolver: None, + suppressed_keys: HashSet::new(), + })), + } + } + + /// Install an already-prepared child environment snapshot. + /// + /// This is intentionally narrower than [`Self::install_environment`]: it + /// updates only the workload-facing env map and clears resolver state so a + /// process that does not own gateway/provider resolver material can still + /// pick up refreshed provider env for future child processes. + pub fn install_child_env_snapshot( + &self, + revision: u64, + mut child_env: HashMap, + ) -> usize { + let mut inner = self + .inner + .write() + .expect("provider credential state poisoned"); + + for key in &inner.suppressed_keys { + child_env.remove(key); + } + + inner.current = Arc::new(ProviderCredentialSnapshot { + revision, + child_env, + dynamic_credentials: HashMap::new(), + }); + inner.generations.clear(); + inner.current_resolver = None; + inner.combined_resolver = None; + inner.current.child_env.len() + } + pub fn snapshot(&self) -> Arc { self.inner .read() @@ -594,6 +651,39 @@ mod tests { ); } + #[test] + fn child_env_snapshot_install_updates_env_without_resolver_material() { + let state = ProviderCredentialState::from_child_env_snapshot( + 1, + HashMap::from([ + ("GITHUB_TOKEN".to_string(), "old".to_string()), + ("GCE_METADATA_HOST".to_string(), "marker".to_string()), + ]), + ); + state.remove_env_key("GCE_METADATA_HOST"); + + let env_count = state.install_child_env_snapshot( + 2, + HashMap::from([ + ("GITHUB_TOKEN".to_string(), "new".to_string()), + ("GCE_METADATA_HOST".to_string(), "marker".to_string()), + ]), + ); + + let snapshot = state.snapshot(); + assert_eq!(snapshot.revision, 2); + assert_eq!(env_count, 1); + assert_eq!( + snapshot.child_env.get("GITHUB_TOKEN").map(String::as_str), + Some("new") + ); + assert!(!snapshot.child_env.contains_key("GCE_METADATA_HOST")); + assert!( + state.resolver().is_none(), + "child-env snapshots must not install provider resolver material" + ); + } + #[test] fn stale_generation_falls_back_to_current_credential_after_retention_window() { let state = ProviderCredentialState::from_environment( diff --git a/crates/openshell-core/src/sandbox_env.rs b/crates/openshell-core/src/sandbox_env.rs index c56a1c889d..f066580636 100644 --- a/crates/openshell-core/src/sandbox_env.rs +++ b/crates/openshell-core/src/sandbox_env.rs @@ -29,6 +29,34 @@ pub const SANDBOX_COMMAND: &str = "OPENSHELL_SANDBOX_COMMAND"; /// Deployment-controlled telemetry toggle propagated to the sandbox supervisor. pub const TELEMETRY_ENABLED: &str = "OPENSHELL_TELEMETRY_ENABLED"; +/// Supervisor pod/runtime topology. Kubernetes sidecar mode sets this to +/// `"sidecar"`; the default combined supervisor path omits it. +pub const SUPERVISOR_TOPOLOGY: &str = "OPENSHELL_SUPERVISOR_TOPOLOGY"; + +/// Network enforcement backend selected by the compute driver. +pub const NETWORK_ENFORCEMENT_MODE: &str = "OPENSHELL_NETWORK_ENFORCEMENT_MODE"; + +/// Whether network policy evaluation must bind requests to the peer binary. +/// +/// The default when unset is `"required"`. Kubernetes sidecar experiments may +/// set this to `"relaxed"` to enforce endpoint and L7 policy without per-binary +/// `/proc` identity binding. +pub const NETWORK_BINARY_IDENTITY: &str = "OPENSHELL_NETWORK_BINARY_IDENTITY"; + +/// Unix socket used by Kubernetes sidecar topology for local coordination. +/// +/// The network sidecar owns gateway credentials and serves policy/provider +/// state over this socket instead of exposing gateway credentials to the agent +/// container. +pub const SIDECAR_CONTROL_SOCKET: &str = "OPENSHELL_SIDECAR_CONTROL_SOCKET"; + +/// Optional TLS server name override used when connecting to the gateway. +pub const GATEWAY_TLS_SERVER_NAME: &str = "OPENSHELL_GATEWAY_TLS_SERVER_NAME"; + +/// Directory where the network supervisor writes the proxy CA files consumed +/// by workload child processes. +pub const PROXY_TLS_DIR: &str = "OPENSHELL_PROXY_TLS_DIR"; + /// Path to the CA certificate for mTLS communication with the gateway. pub const TLS_CA: &str = "OPENSHELL_TLS_CA"; diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index 48ddfb8f11..688660dbb5 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -53,9 +53,43 @@ pods do not need direct external ingress for SSH. ## Container Security Context -The driver grants the sandbox agent container the Linux capabilities the -supervisor needs for namespace setup and policy enforcement. It can also request -a Kubernetes AppArmor profile through `app_armor_profile`. +The default `combined` supervisor topology grants the sandbox agent container +the Linux capabilities the supervisor needs for namespace setup and process, +filesystem, and network policy enforcement. + +The `sidecar` supervisor topology moves pod-level network setup into a root init +container. In the default process/binary-aware mode, the long-lived network +sidecar runs as UID 0 with `allowPrivilegeEscalation: false`, drops default +Linux capabilities, and adds only `SYS_PTRACE` plus `DAC_READ_SEARCH` for +cross-UID workload `/proc` inspection. The agent container also runs as the +resolved sandbox UID/GID with `allowPrivilegeEscalation: false` and +`capabilities.drop: ["ALL"]`. +Set `sidecar.process_binary_aware_network_policy = false` to run the network +sidecar as the configured non-root `sidecar.proxy_uid`, omit the extra `/proc` +inspection capabilities, and enforce endpoint/L7 network policy without +matching `policy.binaries`. +In this mode OpenShell preserves gateway session and SSH behavior, but the +process supervisor does not perform root-to-sandbox privilege dropping or +supervisor identity mount isolation. It still applies Landlock filesystem policy +and child seccomp filters where the kernel/runtime supports them. Network +endpoint and L7 policy remain enforced by the network sidecar, and +sidecar pods use a shared process namespace so the network sidecar can resolve +process/binary identity through `/proc/`. + +Sidecar mode keeps gateway credentials in the network sidecar. The agent +container does not mount the projected service-account token used for sandbox +token bootstrap, does not mount the sandbox client TLS secret, and does not get +gateway callback environment variables. The process supervisor receives policy +and provider environment state from the sidecar over a local control socket in +the shared sidecar state volume. The sidecar accepts only the pre-workload +process-supervisor connection, authenticates its UID/GID/PID with peer +credentials, and removes the listener afterward. SSH relays use a Linux +abstract socket whose peer PID must match that authenticated supervisor. Both +supervisors exit if the control connection closes, coupling their container +restart lifecycle before a new authoritative client can be established. + +The driver can request a Kubernetes AppArmor profile through +`app_armor_profile`. Supported values are `Unconfined`, `RuntimeDefault`, and `Localhost/`. An empty or unset value omits diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 23f80f680b..1eeaac8396 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -15,6 +15,9 @@ pub const DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME: &str = "default"; /// Default storage size for the workspace PVC. pub const DEFAULT_WORKSPACE_STORAGE_SIZE: &str = "2Gi"; +/// Default non-root UID for relaxed Kubernetes network supervisor sidecars. +pub const DEFAULT_PROXY_UID: u32 = 1337; + /// How the supervisor binary is delivered into sandbox pods. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] @@ -59,12 +62,16 @@ pub enum SupervisorTopology { /// Run networking and process supervision in the agent container. #[default] Combined, + /// Run network supervision in a privileged sidecar and process supervision + /// as a low-capability wrapper in the agent container. + Sidecar, } impl std::fmt::Display for SupervisorTopology { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Self::Combined => f.write_str("combined"), + Self::Sidecar => f.write_str("sidecar"), } } } @@ -75,11 +82,49 @@ impl FromStr for SupervisorTopology { fn from_str(s: &str) -> Result { match s { "combined" => Ok(Self::Combined), - other => Err(format!("unknown supervisor topology '{other}'")), + "sidecar" => Ok(Self::Sidecar), + other => Err(format!("unknown topology '{other}'")), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct KubernetesSidecarConfig { + /// UID used by relaxed long-running network sidecars in `sidecar` + /// topology. The network init container installs nftables rules that + /// exempt this UID, so it must not match the sandbox workload UID. + /// Strict process/binary-aware sidecars run as UID 0 so Kubernetes grants + /// the requested `/proc` inspection capabilities into the effective set. + pub proxy_uid: u32, + /// Require process/binary-aware network policy enforcement in sidecar + /// topology. When disabled, the network sidecar runs as `proxy_uid`, + /// drops the extra `/proc` inspection permissions, and evaluates + /// endpoint/L7 policy without matching `policy.binaries`. + pub process_binary_aware_network_policy: bool, +} + +impl Default for KubernetesSidecarConfig { + fn default() -> Self { + Self { + proxy_uid: DEFAULT_PROXY_UID, + process_binary_aware_network_policy: true, } } } +impl KubernetesSidecarConfig { + pub fn validate_proxy_uid(&self) -> Result<(), String> { + if self.proxy_uid < openshell_policy::MIN_SANDBOX_UID { + return Err(format!( + "sidecar.proxy_uid must be at least {}", + openshell_policy::MIN_SANDBOX_UID + )); + } + Ok(()) + } +} + /// Kubernetes `AppArmor` profile requested for the sandbox agent container. #[derive(Debug, Clone, PartialEq, Eq)] pub enum AppArmorProfile { @@ -205,7 +250,9 @@ pub struct KubernetesComputeConfig { /// How the supervisor binary is delivered into sandbox pods. pub supervisor_sideload_method: SupervisorSideloadMethod, /// How the supervisor is arranged for Kubernetes sandbox pods. - pub supervisor_topology: SupervisorTopology, + pub topology: SupervisorTopology, + /// Sidecar-only settings used when `topology = "sidecar"`. + pub sidecar: KubernetesSidecarConfig, pub grpc_endpoint: String, pub ssh_socket_path: String, pub client_tls_secret_name: String, @@ -291,7 +338,8 @@ impl Default for KubernetesComputeConfig { supervisor_image: config::default_supervisor_image(), supervisor_image_pull_policy: String::new(), supervisor_sideload_method: SupervisorSideloadMethod::default(), - supervisor_topology: SupervisorTopology::default(), + topology: SupervisorTopology::default(), + sidecar: KubernetesSidecarConfig::default(), grpc_endpoint: String::new(), ssh_socket_path: "/run/openshell/ssh.sock".to_string(), client_tls_secret_name: String::new(), @@ -336,6 +384,10 @@ impl KubernetesComputeConfig { ) } + pub fn validate_proxy_uid(&self) -> Result<(), String> { + self.sidecar.validate_proxy_uid() + } + /// Resolve the sandbox UID/GID pair. /// /// Resolution order: @@ -351,6 +403,7 @@ impl KubernetesComputeConfig { if let Some(uid) = self.sandbox_uid { return uid; } + // Try OpenShift SCC annotation. if let Some(anns) = namespace_annotations && let Some(range) = anns.get(ANNOTATION_SCC_UID_RANGE) && let Some(uid) = Self::from_open_shift_uid_range(range) @@ -462,39 +515,140 @@ mod tests { } #[test] - fn default_service_account_name_is_default() { + fn default_topology_is_combined() { let cfg = KubernetesComputeConfig::default(); - assert_eq!( - cfg.service_account_name, - DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME - ); + assert_eq!(cfg.topology, SupervisorTopology::Combined); + assert_eq!(cfg.topology.to_string(), "combined"); } #[test] - fn default_supervisor_topology_is_combined() { + fn default_proxy_uid_is_dedicated_non_root_uid() { let cfg = KubernetesComputeConfig::default(); - assert_eq!(cfg.supervisor_topology, SupervisorTopology::Combined); - assert_eq!(cfg.supervisor_topology.to_string(), "combined"); + assert_eq!(cfg.sidecar.proxy_uid, DEFAULT_PROXY_UID); } #[test] - fn serde_override_supervisor_topology_combined() { + fn default_sidecar_requires_process_binary_aware_network_policy() { + let cfg = KubernetesComputeConfig::default(); + assert!(cfg.sidecar.process_binary_aware_network_policy); + } + + #[test] + fn serde_override_topology_sidecar() { let json = serde_json::json!({ - "supervisor_topology": "combined" + "topology": "sidecar" }); let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); - assert_eq!(cfg.supervisor_topology, SupervisorTopology::Combined); + assert_eq!(cfg.topology, SupervisorTopology::Sidecar); } #[test] - fn serde_rejects_invalid_supervisor_topology() { + fn serde_override_topology_combined() { let json = serde_json::json!({ - "supervisor_topology": "unsupported" + "topology": "combined" + }); + let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); + assert_eq!(cfg.topology, SupervisorTopology::Combined); + } + + #[test] + fn serde_rejects_sidecar_binary_identity_field() { + let json = serde_json::json!({ + "sidecar": { + "binary_identity": "shared-pid" + } + }); + let err = serde_json::from_value::(json).unwrap_err(); + assert!(err.to_string().contains("unknown field")); + } + + #[test] + fn serde_override_sidecar_process_binary_aware_network_policy_nested() { + let json = serde_json::json!({ + "sidecar": { + "process_binary_aware_network_policy": false + } + }); + let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); + assert!(!cfg.sidecar.process_binary_aware_network_policy); + } + + #[test] + fn serde_override_sidecar_proxy_uid_nested() { + let json = serde_json::json!({ + "sidecar": { + "proxy_uid": 2000 + } + }); + let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap(); + assert_eq!(cfg.sidecar.proxy_uid, 2000); + cfg.validate_proxy_uid().unwrap(); + } + + #[test] + fn validate_proxy_uid_rejects_privileged_uid() { + let cfg = KubernetesComputeConfig { + sidecar: KubernetesSidecarConfig { + proxy_uid: 999, + ..KubernetesSidecarConfig::default() + }, + ..KubernetesComputeConfig::default() + }; + let err = cfg.validate_proxy_uid().unwrap_err(); + assert!(err.contains("proxy_uid")); + } + + #[test] + fn serde_rejects_invalid_topology() { + let json = serde_json::json!({ + "topology": "unsupported" }); let err = serde_json::from_value::(json).unwrap_err(); assert!(err.to_string().contains("unknown variant")); } + #[test] + fn serde_rejects_removed_topology_alias_field() { + let mut json = serde_json::Map::new(); + json.insert( + ["supervisor", "topology"].join("_"), + serde_json::json!("sidecar"), + ); + let err = + serde_json::from_value::(serde_json::Value::Object(json)) + .unwrap_err(); + assert!(err.to_string().contains("unknown field")); + } + + #[test] + fn serde_rejects_removed_flat_sidecar_fields() { + for json in [ + serde_json::json!({ "sidecar_binary_identity": "shared-pid" }), + serde_json::json!({ "proxy_uid": 2000 }), + ] { + let err = serde_json::from_value::(json).unwrap_err(); + assert!(err.to_string().contains("unknown field")); + } + } + + #[test] + fn serde_rejects_removed_process_enforcement_field() { + let json = serde_json::json!({ + "process_enforcement": "network-only" + }); + let err = serde_json::from_value::(json).unwrap_err(); + assert!(err.to_string().contains("unknown field")); + } + + #[test] + fn default_service_account_name_is_default() { + let cfg = KubernetesComputeConfig::default(); + assert_eq!( + cfg.service_account_name, + DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME + ); + } + #[test] fn serde_override_workspace_storage_size() { let json = serde_json::json!({ diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 166f18b1cc..db7492d27d 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -5,8 +5,9 @@ use super::AppArmorProfile; use crate::config::{ - DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_SANDBOX_UID, DEFAULT_WORKSPACE_STORAGE_SIZE, - KubernetesComputeConfig, SupervisorSideloadMethod, + DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_SANDBOX_UID, + DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, SupervisorSideloadMethod, + SupervisorTopology, }; use futures::{Stream, StreamExt, TryStreamExt}; use k8s_openapi::api::core::v1::{Event as KubeEventObj, Namespace, Node}; @@ -221,6 +222,9 @@ impl KubernetesComputeDriver { config .validate_sandbox_identity_config() .map_err(KubernetesDriverError::Precondition)?; + config + .validate_proxy_uid() + .map_err(KubernetesDriverError::Precondition)?; let base_config = match kube::Config::incluster() { Ok(c) => c, Err(_) => kube::Config::infer() @@ -549,7 +553,8 @@ impl KubernetesComputeDriver { .map_err(KubernetesDriverError::Message)?; // Resolve sandbox UID/GID from config or OpenShift SCC namespace annotations. - let (resolved_uid, resolved_gid, ns_annotations) = self.resolve_sandbox_identity().await; + let (resolved_user_id, resolved_group_id, ns_annotations) = + self.resolve_sandbox_identity().await; let params = SandboxPodParams { default_image: &self.config.default_image, @@ -558,6 +563,12 @@ impl KubernetesComputeDriver { supervisor_image: &self.config.supervisor_image, supervisor_image_pull_policy: &self.config.supervisor_image_pull_policy, supervisor_sideload_method: self.config.supervisor_sideload_method, + topology: self.config.topology, + proxy_uid: self.config.sidecar.proxy_uid, + process_binary_aware_network_policy: self + .config + .sidecar + .process_binary_aware_network_policy, service_account_name: &self.config.service_account_name, sandbox_id: &sandbox.id, sandbox_name: &sandbox.name, @@ -574,9 +585,10 @@ impl KubernetesComputeDriver { provider_spiffe_workload_api_socket_path: &self .config .provider_spiffe_workload_api_socket_path, - sandbox_uid: resolved_uid, - sandbox_gid: resolved_gid, + sandbox_uid: resolved_user_id, + sandbox_gid: resolved_group_id, }; + validate_sidecar_proxy_identity(¶ms)?; let mut obj = DynamicObject::new(name, &agent_sandbox_api.resource); // Copy only the SCC-related annotations onto the Sandbox CR for @@ -1035,6 +1047,32 @@ const SUPERVISOR_VOLUME_NAME: &str = "openshell-supervisor-bin"; /// Name of the init container that installs the supervisor binary. const SUPERVISOR_INIT_CONTAINER_NAME: &str = "openshell-supervisor-install"; +/// Name of the init container that prepares pod-level sidecar networking. +const SUPERVISOR_NETWORK_INIT_CONTAINER_NAME: &str = "openshell-network-init"; + +/// Container name for the network-only supervisor sidecar. +const SUPERVISOR_NETWORK_SIDECAR_NAME: &str = "openshell-supervisor-network"; + +/// UID used by strict process/binary-aware sidecars so Kubernetes grants the +/// requested capability set into the effective set without privilege escalation. +const BINARY_AWARE_SIDECAR_PROXY_UID: u32 = 0; + +/// Shared volume used by the network sidecar and process-only supervisor for +/// local coordination in sidecar topology. +const SIDECAR_STATE_VOLUME_NAME: &str = "openshell-sidecar-state"; +const SIDECAR_STATE_MOUNT_PATH: &str = "/run/openshell-sidecar"; +const SIDECAR_CONTROL_SOCKET: &str = "/run/openshell-sidecar/control.sock"; +// Linux abstract socket names are scoped to the pod's shared network namespace. +// Unlike a filesystem socket in the shared state volume, the workload cannot +// unlink and replace this relay endpoint after the trusted supervisor binds it. +const SIDECAR_SSH_SOCKET_FILE: &str = "@openshell-sidecar-ssh"; + +/// Shared TLS work directory. The network sidecar writes the proxy CA bundle +/// here, while the agent container consumes it after sidecar bootstrap. +const SIDECAR_TLS_VOLUME_NAME: &str = "openshell-supervisor-tls"; +const SIDECAR_TLS_MOUNT_PATH: &str = "/etc/openshell-tls/proxy"; +const SIDECAR_CLIENT_TLS_MOUNT_PATH: &str = "/etc/openshell-tls/proxy/client"; + /// Build the emptyDir volume that holds the supervisor binary. /// /// The init container writes the binary here; the agent container reads it. @@ -1109,31 +1147,12 @@ fn supervisor_init_container( spec } -/// Apply supervisor side-load transforms to an already-built pod template JSON. -/// -/// Depending on the sideload method: -/// - **`ImageVolume`**: mounts the supervisor OCI image directly as a read-only -/// volume (no init container needed, requires K8s >= v1.33). -/// - **`InitContainer`**: injects an emptyDir volume and an init container that -/// copies the supervisor binary from the supervisor image into that volume. -/// -/// In both cases, the agent container gets a command override to run the -/// side-loaded binary and `runAsUser: 0` so it can create network namespaces, -/// set up the proxy, and configure Landlock/seccomp. -#[allow(clippy::similar_names)] -fn apply_supervisor_sideload( - pod_template: &mut serde_json::Value, +fn apply_supervisor_binary_source( + spec: &mut serde_json::Map, supervisor_image: &str, supervisor_image_pull_policy: &str, method: SupervisorSideloadMethod, - sandbox_uid: u32, - sandbox_gid: u32, ) { - let Some(spec) = pod_template.get_mut("spec").and_then(|v| v.as_object_mut()) else { - return; - }; - - // 1. Add the volume (image source or emptyDir depending on method) let volumes = spec .entry("volumes") .or_insert_with(|| serde_json::json!([])) @@ -1152,7 +1171,6 @@ fn apply_supervisor_sideload( } } - // 2. Add the init container only for the init-container method if method == SupervisorSideloadMethod::InitContainer { let init_containers = spec .entry("initContainers") @@ -1165,8 +1183,35 @@ fn apply_supervisor_sideload( )); } } +} + +/// Apply supervisor side-load transforms to an already-built pod template JSON. +/// +/// Depending on the sideload method: +/// - **`ImageVolume`**: mounts the supervisor OCI image directly as a read-only +/// volume (no init container needed, requires K8s >= v1.33). +/// - **`InitContainer`**: injects an emptyDir volume and an init container that +/// copies the supervisor binary from the supervisor image into that volume. +/// +/// In both cases, the agent container gets a command override to run the +/// side-loaded binary as root so it can create network namespaces, set up the +/// proxy, and configure Landlock/seccomp. +#[allow(clippy::similar_names)] +fn apply_supervisor_sideload( + pod_template: &mut serde_json::Value, + supervisor_image: &str, + supervisor_image_pull_policy: &str, + method: SupervisorSideloadMethod, + sandbox_uid: u32, + sandbox_gid: u32, +) { + let Some(spec) = pod_template.get_mut("spec").and_then(|v| v.as_object_mut()) else { + return; + }; + + apply_supervisor_binary_source(spec, supervisor_image, supervisor_image_pull_policy, method); - // 3. Find the agent container and add volume mount + command override + // Find the agent container and add volume mount + command override let Some(containers) = spec.get_mut("containers").and_then(|v| v.as_array_mut()) else { return; }; @@ -1227,6 +1272,398 @@ fn apply_supervisor_sideload( } } +fn sidecar_state_volume_mount() -> serde_json::Value { + serde_json::json!({ + "name": SIDECAR_STATE_VOLUME_NAME, + "mountPath": SIDECAR_STATE_MOUNT_PATH, + }) +} + +fn sidecar_tls_volume_mount() -> serde_json::Value { + serde_json::json!({ + "name": SIDECAR_TLS_VOLUME_NAME, + "mountPath": SIDECAR_TLS_MOUNT_PATH, + }) +} + +fn copy_log_level_env( + env: &mut Vec, + template_environment: &std::collections::HashMap, + spec_environment: &std::collections::HashMap, +) { + if let Some(value) = spec_environment + .get(openshell_core::sandbox_env::LOG_LEVEL) + .or_else(|| template_environment.get(openshell_core::sandbox_env::LOG_LEVEL)) + { + upsert_env(env, openshell_core::sandbox_env::LOG_LEVEL, value); + } +} + +fn supervisor_sidecar_env( + template_environment: &std::collections::HashMap, + spec_environment: &std::collections::HashMap, + params: &SandboxPodParams<'_>, +) -> Vec { + let mut env = Vec::new(); + apply_required_env( + &mut env, + params.sandbox_id, + params.sandbox_name, + params.grpc_endpoint, + "", + !params.client_tls_secret_name.is_empty(), + provider_spiffe_socket_path(params), + ); + if !params.client_tls_secret_name.is_empty() { + upsert_env( + &mut env, + openshell_core::sandbox_env::TLS_CA, + &format!("{SIDECAR_CLIENT_TLS_MOUNT_PATH}/ca.crt"), + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::TLS_CERT, + &format!("{SIDECAR_CLIENT_TLS_MOUNT_PATH}/tls.crt"), + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::TLS_KEY, + &format!("{SIDECAR_CLIENT_TLS_MOUNT_PATH}/tls.key"), + ); + } + copy_log_level_env(&mut env, template_environment, spec_environment); + upsert_env( + &mut env, + openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY, + "sidecar", + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE, + "sidecar-nftables", + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET, + SIDECAR_CONTROL_SOCKET, + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::SSH_SOCKET_PATH, + SIDECAR_SSH_SOCKET_FILE, + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::PROXY_TLS_DIR, + SIDECAR_TLS_MOUNT_PATH, + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::SANDBOX_UID, + ¶ms.sandbox_uid.to_string(), + ); + upsert_env( + &mut env, + openshell_core::sandbox_env::SANDBOX_GID, + ¶ms.sandbox_gid.to_string(), + ); + if !params.process_binary_aware_network_policy { + upsert_env( + &mut env, + openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY, + "relaxed", + ); + } + env +} + +fn supervisor_sidecar_container( + template_environment: &std::collections::HashMap, + spec_environment: &std::collections::HashMap, + params: &SandboxPodParams<'_>, +) -> serde_json::Value { + let proxy_uid = effective_sidecar_proxy_uid(params); + let capabilities = if params.process_binary_aware_network_policy { + serde_json::json!({ + "drop": ["ALL"], + "add": ["SYS_PTRACE", "DAC_READ_SEARCH"] + }) + } else { + serde_json::json!({ + "drop": ["ALL"] + }) + }; + let mut container = serde_json::json!({ + "name": SUPERVISOR_NETWORK_SIDECAR_NAME, + "image": params.supervisor_image, + "command": [ + SUPERVISOR_IMAGE_BINARY_PATH, + "--mode=network", + ], + "env": supervisor_sidecar_env(template_environment, spec_environment, params), + "securityContext": { + "runAsUser": proxy_uid, + "runAsGroup": params.sandbox_gid, + "runAsNonRoot": proxy_uid != 0, + "allowPrivilegeEscalation": false, + "capabilities": capabilities + }, + "volumeMounts": [ + sidecar_state_volume_mount(), + sidecar_tls_volume_mount(), + { + "name": "openshell-sa-token", + "mountPath": "/var/run/secrets/openshell", + "readOnly": true + } + ] + }); + if !params.supervisor_image_pull_policy.is_empty() { + container["imagePullPolicy"] = serde_json::json!(params.supervisor_image_pull_policy); + } + if params.provider_spiffe_enabled { + container["volumeMounts"] + .as_array_mut() + .expect("volumeMounts is an array") + .push(serde_json::json!({ + "name": SPIFFE_WORKLOAD_API_VOLUME_NAME, + "mountPath": spiffe_socket_mount_path(params.provider_spiffe_workload_api_socket_path), + "readOnly": true, + })); + } + if let Some(profile) = params.app_armor_profile { + container["securityContext"]["appArmorProfile"] = app_armor_profile_to_k8s(profile); + } + container +} + +fn effective_sidecar_proxy_uid(params: &SandboxPodParams<'_>) -> u32 { + if params.process_binary_aware_network_policy { + BINARY_AWARE_SIDECAR_PROXY_UID + } else { + params.proxy_uid + } +} + +fn supervisor_network_init_container(params: &SandboxPodParams<'_>) -> serde_json::Value { + let proxy_uid = effective_sidecar_proxy_uid(params); + let mut container = serde_json::json!({ + "name": SUPERVISOR_NETWORK_INIT_CONTAINER_NAME, + "image": params.supervisor_image, + "command": [ + SUPERVISOR_IMAGE_BINARY_PATH, + "--mode=network-init", + "--proxy-uid", + proxy_uid.to_string(), + "--proxy-gid", + params.sandbox_gid.to_string(), + "--sidecar-state-dir", + SIDECAR_STATE_MOUNT_PATH, + "--sidecar-tls-dir", + SIDECAR_TLS_MOUNT_PATH, + ], + "securityContext": { + "runAsUser": 0, + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": ["ALL"], + "add": ["NET_ADMIN", "NET_RAW", "CHOWN", "FOWNER"] + } + }, + "volumeMounts": [ + sidecar_state_volume_mount(), + sidecar_tls_volume_mount(), + ] + }); + if !params.supervisor_image_pull_policy.is_empty() { + container["imagePullPolicy"] = serde_json::json!(params.supervisor_image_pull_policy); + } + if !params.client_tls_secret_name.is_empty() { + container["volumeMounts"] + .as_array_mut() + .expect("volumeMounts is an array") + .push(serde_json::json!({ + "name": "openshell-client-tls", + "mountPath": "/etc/openshell-tls/client", + "readOnly": true + })); + } + if let Some(profile) = params.app_armor_profile { + container["securityContext"]["appArmorProfile"] = app_armor_profile_to_k8s(profile); + } + container +} + +fn apply_supervisor_sidecar_topology( + pod_template: &mut serde_json::Value, + template_environment: &std::collections::HashMap, + spec_environment: &std::collections::HashMap, + params: &SandboxPodParams<'_>, +) { + let Some(spec) = pod_template.get_mut("spec").and_then(|v| v.as_object_mut()) else { + return; + }; + + let pod_security_context = spec + .entry("securityContext") + .or_insert_with(|| serde_json::json!({})); + if let Some(sc) = pod_security_context.as_object_mut() { + sc.insert("fsGroup".to_string(), serde_json::json!(params.sandbox_gid)); + } + + spec.insert("shareProcessNamespace".to_string(), serde_json::json!(true)); + + apply_supervisor_binary_source( + spec, + params.supervisor_image, + params.supervisor_image_pull_policy, + params.supervisor_sideload_method, + ); + + let volumes = spec + .entry("volumes") + .or_insert_with(|| serde_json::json!([])) + .as_array_mut(); + if let Some(volumes) = volumes { + volumes.push(serde_json::json!({ + "name": SIDECAR_STATE_VOLUME_NAME, + "emptyDir": {} + })); + volumes.push(serde_json::json!({ + "name": SIDECAR_TLS_VOLUME_NAME, + "emptyDir": {} + })); + } + + let init_containers = spec + .entry("initContainers") + .or_insert_with(|| serde_json::json!([])) + .as_array_mut(); + if let Some(init_containers) = init_containers { + init_containers.push(supervisor_network_init_container(params)); + } + + let Some(containers) = spec.get_mut("containers").and_then(|v| v.as_array_mut()) else { + return; + }; + + let target_index = containers + .iter() + .position(|c| c.get("name").and_then(|v| v.as_str()) == Some("agent")) + .unwrap_or(0); + + if let Some(container) = containers + .get_mut(target_index) + .and_then(|v| v.as_object_mut()) + { + container.insert( + "command".to_string(), + serde_json::json!([ + format!("{}/openshell-sandbox", SUPERVISOR_MOUNT_PATH), + "--mode=process" + ]), + ); + + let security_context = container + .entry("securityContext") + .or_insert_with(|| serde_json::json!({})); + if let Some(sc) = security_context.as_object_mut() { + sc.insert( + "runAsUser".to_string(), + serde_json::json!(params.sandbox_uid), + ); + sc.insert( + "runAsGroup".to_string(), + serde_json::json!(params.sandbox_gid), + ); + sc.insert("runAsNonRoot".to_string(), serde_json::json!(true)); + sc.insert( + "allowPrivilegeEscalation".to_string(), + serde_json::json!(false), + ); + sc.insert( + "capabilities".to_string(), + serde_json::json!({ + "drop": ["ALL"] + }), + ); + } + + let volume_mounts = container + .entry("volumeMounts") + .or_insert_with(|| serde_json::json!([])) + .as_array_mut(); + if let Some(volume_mounts) = volume_mounts { + remove_volume_mount(volume_mounts, "openshell-sa-token"); + remove_volume_mount(volume_mounts, "openshell-client-tls"); + remove_volume_mount(volume_mounts, SPIFFE_WORKLOAD_API_VOLUME_NAME); + volume_mounts.push(supervisor_volume_mount()); + volume_mounts.push(sidecar_state_volume_mount()); + volume_mounts.push(sidecar_tls_volume_mount()); + } + + let env = container + .entry("env") + .or_insert_with(|| serde_json::json!([])) + .as_array_mut(); + if let Some(env) = env { + remove_env(env, openshell_core::sandbox_env::ENDPOINT); + remove_env(env, openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME); + remove_env(env, openshell_core::sandbox_env::TLS_CA); + remove_env(env, openshell_core::sandbox_env::TLS_CERT); + remove_env(env, openshell_core::sandbox_env::TLS_KEY); + remove_env(env, openshell_core::sandbox_env::SANDBOX_TOKEN); + remove_env(env, openshell_core::sandbox_env::SANDBOX_TOKEN_FILE); + remove_env(env, openshell_core::sandbox_env::K8S_SA_TOKEN_FILE); + remove_env( + env, + openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, + ); + upsert_env( + env, + openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY, + "sidecar", + ); + upsert_env( + env, + openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE, + "sidecar-nftables", + ); + upsert_env( + env, + openshell_core::sandbox_env::SSH_SOCKET_PATH, + SIDECAR_SSH_SOCKET_FILE, + ); + upsert_env( + env, + openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET, + SIDECAR_CONTROL_SOCKET, + ); + upsert_env( + env, + openshell_core::sandbox_env::PROXY_TLS_DIR, + SIDECAR_TLS_MOUNT_PATH, + ); + upsert_env( + env, + openshell_core::sandbox_env::SANDBOX_UID, + ¶ms.sandbox_uid.to_string(), + ); + upsert_env( + env, + openshell_core::sandbox_env::SANDBOX_GID, + ¶ms.sandbox_gid.to_string(), + ); + } + } + + containers.push(supervisor_sidecar_container( + template_environment, + spec_environment, + params, + )); +} + /// Apply workspace persistence transforms to an already-built pod template. /// /// This injects: @@ -1242,6 +1679,7 @@ fn apply_supervisor_sideload( /// The init container mounts the PVC at a temporary path so it can still see /// the image's `/sandbox` directory. It checks for a sentinel file and skips /// the copy if the PVC was already initialised. +#[allow(clippy::similar_names)] fn apply_workspace_persistence( pod_template: &mut serde_json::Value, image: &str, @@ -1301,6 +1739,10 @@ fn apply_workspace_persistence( // self-referential symlinks under `/sandbox/.uv`, and GNU cp can // fail while seeding the PVC even though preserving the symlink as-is // is valid. `tar` copies the tree without dereferencing those links. + // Archive only the contents, not the `/sandbox` directory entry + // itself, so extraction never tries to chmod the PVC mount root. + // Extract without restoring owner, mode, or timestamps so the + // non-root init container can seed kubelet-owned PVCs. // // The inner `[ -d ... ]` guard handles custom images that don't have // a /sandbox directory — the copy is skipped but the sentinel is @@ -1308,7 +1750,12 @@ fn apply_workspace_persistence( let copy_cmd = format!( "if [ ! -f {WORKSPACE_INIT_MOUNT_PATH}/{WORKSPACE_SENTINEL} ]; then \ if [ -d {WORKSPACE_MOUNT_PATH} ]; then \ - tar -C {WORKSPACE_MOUNT_PATH} -cf - . | tar -C {WORKSPACE_INIT_MOUNT_PATH} -xpf -; \ + tmp=$(mktemp) && rm -f \"$tmp\" && \ + (cd {WORKSPACE_MOUNT_PATH} && find . -mindepth 1 -maxdepth 1 -exec tar -cf \"$tmp\" {{}} +) && \ + if [ -f \"$tmp\" ]; then \ + tar -C {WORKSPACE_INIT_MOUNT_PATH} --no-same-owner --no-same-permissions --touch -xf \"$tmp\" && \ + rm -f \"$tmp\"; \ + fi; \ fi && \ touch {WORKSPACE_INIT_MOUNT_PATH}/{WORKSPACE_SENTINEL}; \ fi" @@ -1366,6 +1813,9 @@ struct SandboxPodParams<'a> { supervisor_image: &'a str, supervisor_image_pull_policy: &'a str, supervisor_sideload_method: SupervisorSideloadMethod, + topology: SupervisorTopology, + proxy_uid: u32, + process_binary_aware_network_policy: bool, service_account_name: &'a str, sandbox_id: &'a str, sandbox_name: &'a str, @@ -1397,6 +1847,9 @@ impl Default for SandboxPodParams<'_> { supervisor_image: "", supervisor_image_pull_policy: "", supervisor_sideload_method: SupervisorSideloadMethod::default(), + topology: SupervisorTopology::default(), + proxy_uid: DEFAULT_PROXY_UID, + process_binary_aware_network_policy: true, service_account_name: DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, sandbox_id: "", sandbox_name: "", @@ -1417,6 +1870,18 @@ impl Default for SandboxPodParams<'_> { } } +fn validate_sidecar_proxy_identity( + params: &SandboxPodParams<'_>, +) -> Result<(), KubernetesDriverError> { + if params.topology == SupervisorTopology::Sidecar && params.proxy_uid == params.sandbox_uid { + return Err(KubernetesDriverError::Precondition(format!( + "proxy_uid ({}) must not match sandbox_uid ({}) in sidecar topology", + params.proxy_uid, params.sandbox_uid + ))); + } + Ok(()) +} + fn spec_pod_env(spec: Option<&SandboxSpec>) -> std::collections::HashMap { let mut env = spec.map_or_else(Default::default, |s| s.environment.clone()); if let Some(s) = spec.filter(|s| !s.log_level.is_empty()) { @@ -1707,13 +2172,22 @@ fn sandbox_template_to_k8s_with_gpu_requirements( serde_json::Value::Array(vec![serde_json::Value::Object(container)]), ); - // Add TLS secret volume. Mode 0400 (owner-read) prevents the - // unprivileged sandbox user from reading the mTLS private key. + // Add TLS secret volume. Combined mode uses mode 0400 because the + // supervisor starts as root and drops privileges before running workload + // children. Sidecar mode keeps the process supervisor non-root, so it uses + // pod fsGroup + 0440 to preserve gateway session and SSH control behavior. let mut volumes: Vec = Vec::new(); if !params.client_tls_secret_name.is_empty() { + let client_tls_default_mode = match params.topology { + SupervisorTopology::Combined => 0o400, + SupervisorTopology::Sidecar => 0o440, + }; volumes.push(serde_json::json!({ "name": "openshell-client-tls", - "secret": { "secretName": params.client_tls_secret_name, "defaultMode": 256 } + "secret": { + "secretName": params.client_tls_secret_name, + "defaultMode": client_tls_default_mode + } })); } if params.provider_spiffe_enabled { @@ -1728,7 +2202,12 @@ fn sandbox_template_to_k8s_with_gpu_requirements( // Projected ServiceAccountToken volume — kubelet writes a short-lived // audience-bound JWT into /var/run/secrets/openshell/token and rotates // it automatically. The supervisor exchanges this for a gateway-minted - // JWT via `IssueSandboxToken` once at startup. + // JWT via `IssueSandboxToken` once at startup. In sidecar topology both + // supervisor containers run with the sandbox GID and need group-read access. + let sa_token_default_mode = match params.topology { + SupervisorTopology::Combined => 0o400, + SupervisorTopology::Sidecar => 0o440, + }; volumes.push(serde_json::json!({ "name": "openshell-sa-token", "projected": { @@ -1739,7 +2218,7 @@ fn sandbox_template_to_k8s_with_gpu_requirements( "path": "token" } }], - "defaultMode": 256 + "defaultMode": sa_token_default_mode } })); spec.insert("volumes".to_string(), serde_json::Value::Array(volumes)); @@ -1763,14 +2242,26 @@ fn sandbox_template_to_k8s_with_gpu_requirements( let mut result = serde_json::Value::Object(template_value); - apply_supervisor_sideload( - &mut result, - params.supervisor_image, - params.supervisor_image_pull_policy, - params.supervisor_sideload_method, - params.sandbox_uid, - params.sandbox_gid, - ); + match params.topology { + SupervisorTopology::Combined => { + apply_supervisor_sideload( + &mut result, + params.supervisor_image, + params.supervisor_image_pull_policy, + params.supervisor_sideload_method, + params.sandbox_uid, + params.sandbox_gid, + ); + } + SupervisorTopology::Sidecar => { + apply_supervisor_sidecar_topology( + &mut result, + &template.environment, + spec_environment, + params, + ); + } + } // Inject workspace persistence (init container + PVC volume mount) so // that /sandbox data survives pod rescheduling. Skipped when the user @@ -2088,6 +2579,14 @@ fn upsert_env(env: &mut Vec, name: &str, value: &str) { env.push(serde_json::json!({"name": name, "value": value})); } +fn remove_env(env: &mut Vec, name: &str) { + env.retain(|item| item.get("name").and_then(|value| value.as_str()) != Some(name)); +} + +fn remove_volume_mount(volume_mounts: &mut Vec, name: &str) { + volume_mounts.retain(|mount| mount.get("name").and_then(|value| value.as_str()) != Some(name)); +} + /// Extract a string value from the template's `platform_config` Struct. fn platform_config_string(template: &SandboxTemplate, key: &str) -> Option { let config = template.platform_config.as_ref()?; @@ -2262,6 +2761,15 @@ mod tests { assert!(!should_try_next_sandbox_api_version(&err)); } + fn rendered_env<'a>(container: &'a serde_json::Value, name: &str) -> Option<&'a str> { + container["env"] + .as_array()? + .iter() + .find(|item| item.get("name").and_then(|value| value.as_str()) == Some(name))? + .get("value")? + .as_str() + } + #[test] fn driver_config_rejects_invalid_shape() { let template = SandboxTemplate { @@ -2601,6 +3109,385 @@ mod tests { ); } + #[test] + fn sidecar_topology_renders_process_agent_and_network_sidecar() { + let params = SandboxPodParams { + topology: SupervisorTopology::Sidecar, + supervisor_sideload_method: SupervisorSideloadMethod::InitContainer, + supervisor_image: "supervisor-image:latest", + supervisor_image_pull_policy: "IfNotPresent", + grpc_endpoint: "https://openshell-gateway.openshell.svc:8080", + client_tls_secret_name: "openshell-client-tls", + proxy_uid: 2200, + sandbox_uid: 1500, + sandbox_gid: 1500, + ..SandboxPodParams::default() + }; + let pod_template = sandbox_template_to_k8s( + &SandboxTemplate { + image: "agent-image:latest".to_string(), + ..SandboxTemplate::default() + }, + false, + &std::collections::HashMap::new(), + false, + ¶ms, + ); + + assert_eq!(pod_template["spec"]["shareProcessNamespace"], true); + assert_eq!(pod_template["spec"]["securityContext"]["fsGroup"], 1500); + let containers = pod_template["spec"]["containers"].as_array().unwrap(); + assert_eq!(containers.len(), 2); + + let agent = containers + .iter() + .find(|container| container["name"] == "agent") + .unwrap(); + assert_eq!( + agent["command"], + serde_json::json!([ + format!("{SUPERVISOR_MOUNT_PATH}/openshell-sandbox"), + "--mode=process" + ]) + ); + assert_eq!(agent["securityContext"]["runAsUser"], 1500); + assert_eq!(agent["securityContext"]["runAsGroup"], 1500); + assert_eq!(agent["securityContext"]["runAsNonRoot"], true); + assert_eq!(agent["securityContext"]["allowPrivilegeEscalation"], false); + assert_eq!( + agent["securityContext"]["capabilities"], + serde_json::json!({ + "drop": ["ALL"] + }) + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::ENDPOINT), + None + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::GATEWAY_TLS_SERVER_NAME), + None + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::TLS_CA), + None + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::K8S_SA_TOKEN_FILE), + None + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::SSH_SOCKET_PATH), + Some(SIDECAR_SSH_SOCKET_FILE) + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET), + Some(SIDECAR_CONTROL_SOCKET) + ); + assert_eq!(rendered_env(agent, "OPENSHELL_SUPERVISOR_READY_FILE"), None); + assert_eq!(rendered_env(agent, "OPENSHELL_ENTRYPOINT_PID_FILE"), None); + assert_eq!( + rendered_env(agent, "OPENSHELL_SIDECAR_POLICY_SNAPSHOT_FILE"), + None + ); + assert_eq!( + rendered_env(agent, "OPENSHELL_SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE"), + None + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::PROXY_TLS_DIR), + Some(SIDECAR_TLS_MOUNT_PATH) + ); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::SANDBOX_UID), + Some("1500") + ); + + let sidecar = containers + .iter() + .find(|container| container["name"] == SUPERVISOR_NETWORK_SIDECAR_NAME) + .unwrap(); + assert_eq!(sidecar["image"], "supervisor-image:latest"); + assert_eq!(sidecar["imagePullPolicy"], "IfNotPresent"); + assert_eq!( + sidecar["command"], + serde_json::json!([SUPERVISOR_IMAGE_BINARY_PATH, "--mode=network"]) + ); + assert_eq!(sidecar["securityContext"]["runAsUser"], 0); + assert_eq!(sidecar["securityContext"]["runAsGroup"], 1500); + assert_eq!(sidecar["securityContext"]["runAsNonRoot"], false); + assert_eq!( + sidecar["securityContext"]["allowPrivilegeEscalation"], + false + ); + assert_eq!( + sidecar["securityContext"]["capabilities"], + serde_json::json!({ + "drop": ["ALL"], + "add": ["SYS_PTRACE", "DAC_READ_SEARCH"] + }) + ); + assert_eq!( + rendered_env(sidecar, openshell_core::sandbox_env::ENDPOINT), + Some("https://openshell-gateway.openshell.svc:8080") + ); + assert_eq!( + rendered_env(sidecar, openshell_core::sandbox_env::SSH_SOCKET_PATH), + Some(SIDECAR_SSH_SOCKET_FILE) + ); + assert!( + SIDECAR_SSH_SOCKET_FILE.starts_with('@'), + "sidecar SSH relay must use a Linux abstract socket" + ); + assert_eq!( + rendered_env(sidecar, openshell_core::sandbox_env::SANDBOX_UID), + Some("1500") + ); + assert_eq!( + rendered_env(sidecar, openshell_core::sandbox_env::SANDBOX_GID), + Some("1500") + ); + assert_eq!( + rendered_env(sidecar, openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET), + Some(SIDECAR_CONTROL_SOCKET) + ); + assert_eq!( + rendered_env(sidecar, "OPENSHELL_SIDECAR_POLICY_SNAPSHOT_FILE"), + None + ); + assert_eq!( + rendered_env(sidecar, "OPENSHELL_SIDECAR_PROVIDER_ENV_SNAPSHOT_FILE"), + None + ); + assert_eq!( + rendered_env( + sidecar, + openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY + ), + None + ); + assert_eq!(rendered_env(sidecar, "OPENSHELL_ENTRYPOINT_PID_FILE"), None); + assert_eq!( + rendered_env(sidecar, openshell_core::sandbox_env::PROXY_TLS_DIR), + Some(SIDECAR_TLS_MOUNT_PATH) + ); + assert_eq!( + rendered_env(sidecar, openshell_core::sandbox_env::TLS_CA), + Some("/etc/openshell-tls/proxy/client/ca.crt") + ); + let sidecar_mounts = sidecar["volumeMounts"].as_array().unwrap(); + assert!( + !sidecar_mounts + .iter() + .any(|mount| mount["name"] == "openshell-client-tls"), + "runtime sidecar should use the init-copied TLS files, not the root-owned Secret mount" + ); + let agent_mounts = agent["volumeMounts"].as_array().unwrap(); + assert!( + !agent_mounts + .iter() + .any(|mount| mount["name"] == "openshell-sa-token"), + "agent container must not mount gateway bootstrap token in sidecar topology" + ); + assert!( + !agent_mounts + .iter() + .any(|mount| mount["name"] == "openshell-client-tls"), + "agent container must not mount gateway client TLS secret in sidecar topology" + ); + let volumes = pod_template["spec"]["volumes"].as_array().unwrap(); + let sa_token = volumes + .iter() + .find(|volume| volume["name"] == "openshell-sa-token") + .unwrap(); + assert_eq!(sa_token["projected"]["defaultMode"], 0o440); + let client_tls = volumes + .iter() + .find(|volume| volume["name"] == "openshell-client-tls") + .unwrap(); + assert_eq!(client_tls["secret"]["defaultMode"], 0o440); + + let init_containers = pod_template["spec"]["initContainers"].as_array().unwrap(); + let network_init = init_containers + .iter() + .find(|container| container["name"] == SUPERVISOR_NETWORK_INIT_CONTAINER_NAME) + .unwrap(); + assert_eq!(network_init["image"], "supervisor-image:latest"); + assert_eq!(network_init["imagePullPolicy"], "IfNotPresent"); + assert_eq!( + network_init["command"], + serde_json::json!([ + SUPERVISOR_IMAGE_BINARY_PATH, + "--mode=network-init", + "--proxy-uid", + "0", + "--proxy-gid", + "1500", + "--sidecar-state-dir", + SIDECAR_STATE_MOUNT_PATH, + "--sidecar-tls-dir", + SIDECAR_TLS_MOUNT_PATH + ]) + ); + assert_eq!( + network_init["securityContext"]["capabilities"], + serde_json::json!({ + "drop": ["ALL"], + "add": ["NET_ADMIN", "NET_RAW", "CHOWN", "FOWNER"] + }) + ); + let network_init_mounts = network_init["volumeMounts"].as_array().unwrap(); + assert!(network_init_mounts.iter().any(|mount| { + mount["name"] == "openshell-client-tls" + && mount["mountPath"] == "/etc/openshell-tls/client" + })); + } + + #[test] + fn sidecar_topology_can_relax_process_binary_aware_network_policy() { + let params = SandboxPodParams { + topology: SupervisorTopology::Sidecar, + supervisor_sideload_method: SupervisorSideloadMethod::InitContainer, + supervisor_image: "supervisor-image:latest", + proxy_uid: 2200, + sandbox_uid: 1500, + sandbox_gid: 1500, + process_binary_aware_network_policy: false, + ..SandboxPodParams::default() + }; + let pod_template = sandbox_template_to_k8s( + &SandboxTemplate { + image: "agent-image:latest".to_string(), + ..SandboxTemplate::default() + }, + false, + &std::collections::HashMap::new(), + false, + ¶ms, + ); + + let containers = pod_template["spec"]["containers"].as_array().unwrap(); + let sidecar = containers + .iter() + .find(|container| container["name"] == SUPERVISOR_NETWORK_SIDECAR_NAME) + .unwrap(); + assert_eq!(sidecar["securityContext"]["runAsUser"], 2200); + assert_eq!(sidecar["securityContext"]["runAsGroup"], 1500); + assert_eq!(sidecar["securityContext"]["runAsNonRoot"], true); + assert_eq!( + sidecar["securityContext"]["allowPrivilegeEscalation"], + false + ); + assert_eq!( + sidecar["securityContext"]["capabilities"], + serde_json::json!({ + "drop": ["ALL"] + }) + ); + assert_eq!( + rendered_env( + sidecar, + openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY + ), + Some("relaxed") + ); + let init_containers = pod_template["spec"]["initContainers"].as_array().unwrap(); + let network_init = init_containers + .iter() + .find(|container| container["name"] == SUPERVISOR_NETWORK_INIT_CONTAINER_NAME) + .unwrap(); + assert_eq!(network_init["command"][3], "2200"); + } + + #[test] + fn sidecar_topology_adds_shared_state_and_tls_volumes() { + let params = SandboxPodParams { + topology: SupervisorTopology::Sidecar, + supervisor_sideload_method: SupervisorSideloadMethod::ImageVolume, + supervisor_image: "supervisor-image:latest", + grpc_endpoint: "http://openshell-gateway.openshell.svc:8080", + ..SandboxPodParams::default() + }; + let pod_template = sandbox_template_to_k8s( + &SandboxTemplate::default(), + false, + &std::collections::HashMap::new(), + false, + ¶ms, + ); + + let volumes = pod_template["spec"]["volumes"].as_array().unwrap(); + assert!( + volumes + .iter() + .any(|volume| volume["name"] == SIDECAR_STATE_VOLUME_NAME) + ); + assert!( + volumes + .iter() + .any(|volume| volume["name"] == SIDECAR_TLS_VOLUME_NAME) + ); + assert!(volumes.iter().any(|volume| { + volume["name"] == SUPERVISOR_VOLUME_NAME && volume["image"].is_object() + })); + + let containers = pod_template["spec"]["containers"].as_array().unwrap(); + let sidecar = containers + .iter() + .find(|container| container["name"] == SUPERVISOR_NETWORK_SIDECAR_NAME) + .unwrap(); + assert_eq!( + sidecar["securityContext"]["capabilities"], + serde_json::json!({ + "drop": ["ALL"], + "add": ["SYS_PTRACE", "DAC_READ_SEARCH"] + }) + ); + assert_eq!(sidecar["securityContext"]["runAsUser"], 0); + assert_eq!(sidecar["securityContext"]["runAsGroup"], 1000); + assert_eq!(sidecar["securityContext"]["runAsNonRoot"], false); + assert_eq!( + sidecar["securityContext"]["allowPrivilegeEscalation"], + false + ); + + for container_name in ["agent", SUPERVISOR_NETWORK_SIDECAR_NAME] { + let container = containers + .iter() + .find(|container| container["name"] == container_name) + .unwrap(); + let mounts = container["volumeMounts"].as_array().unwrap(); + assert!(mounts.iter().any(|mount| { + mount["name"] == SIDECAR_STATE_VOLUME_NAME + && mount["mountPath"] == SIDECAR_STATE_MOUNT_PATH + })); + assert!(mounts.iter().any(|mount| { + mount["name"] == SIDECAR_TLS_VOLUME_NAME + && mount["mountPath"] == SIDECAR_TLS_MOUNT_PATH + })); + } + let init_containers = pod_template["spec"]["initContainers"].as_array().unwrap(); + let network_init = init_containers + .iter() + .find(|container| container["name"] == SUPERVISOR_NETWORK_INIT_CONTAINER_NAME) + .unwrap(); + assert_eq!(network_init["command"][3], "0"); + } + + #[test] + fn sidecar_topology_rejects_proxy_uid_matching_sandbox_uid() { + let params = SandboxPodParams { + topology: SupervisorTopology::Sidecar, + proxy_uid: 1500, + sandbox_uid: 1500, + ..SandboxPodParams::default() + }; + + let err = validate_sidecar_proxy_identity(¶ms).unwrap_err(); + assert!(matches!(err, KubernetesDriverError::Precondition(_))); + assert!(err.to_string().contains("proxy_uid")); + } + /// Regression test: TLS mount path must match env var paths. /// The volume is mounted at a specific path and the env vars must point to /// files within that same path, otherwise the sandbox will fail to start @@ -3179,6 +4066,16 @@ mod tests { script.contains("tar -C"), "init script must seed image contents with a tar stream" ); + assert!( + script.contains("find . -mindepth 1 -maxdepth 1"), + "init script must archive sandbox contents without the mount root entry" + ); + assert!( + script.contains("--no-same-owner") + && script.contains("--no-same-permissions") + && script.contains("--touch"), + "init script must avoid restoring metadata onto the PVC root" + ); } #[test] diff --git a/crates/openshell-driver-kubernetes/src/lib.rs b/crates/openshell-driver-kubernetes/src/lib.rs index 953ed4abdf..7c56c8de5b 100644 --- a/crates/openshell-driver-kubernetes/src/lib.rs +++ b/crates/openshell-driver-kubernetes/src/lib.rs @@ -6,8 +6,9 @@ pub mod driver; pub mod grpc; pub use config::{ - AppArmorProfile, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_WORKSPACE_STORAGE_SIZE, - KubernetesComputeConfig, SupervisorSideloadMethod, SupervisorTopology, + AppArmorProfile, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, + DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, KubernetesSidecarConfig, + SupervisorSideloadMethod, SupervisorTopology, }; pub use driver::{KubernetesComputeDriver, KubernetesDriverError}; pub use grpc::ComputeDriverService; diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index a300fe6a07..c733b8a45b 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use clap::Parser; +use clap::{ArgAction, Parser}; use miette::{IntoDiagnostic, Result}; use std::net::SocketAddr; use tracing::info; @@ -10,8 +10,9 @@ use tracing_subscriber::EnvFilter; use openshell_core::VERSION; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; use openshell_driver_kubernetes::{ - AppArmorProfile, ComputeDriverService, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, - KubernetesComputeConfig, KubernetesComputeDriver, SupervisorSideloadMethod, SupervisorTopology, + AppArmorProfile, ComputeDriverService, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, + KubernetesComputeConfig, KubernetesComputeDriver, KubernetesSidecarConfig, + SupervisorSideloadMethod, SupervisorTopology, }; #[derive(Parser, Debug)] @@ -80,12 +81,24 @@ struct Args { )] supervisor_sideload_method: SupervisorSideloadMethod, + #[arg(long, env = "OPENSHELL_K8S_TOPOLOGY", default_value = "combined")] + topology: SupervisorTopology, + #[arg( - long, - env = "OPENSHELL_SUPERVISOR_TOPOLOGY", - default_value = "combined" + long = "sidecar-proxy-uid", + alias = "proxy-uid", + env = "OPENSHELL_K8S_SIDECAR_PROXY_UID", + default_value_t = DEFAULT_PROXY_UID + )] + sidecar_proxy_uid: u32, + + #[arg( + long = "sidecar-process-binary-aware-network-policy", + env = "OPENSHELL_K8S_SIDECAR_PROCESS_BINARY_AWARE_NETWORK_POLICY", + default_value_t = true, + action = ArgAction::Set )] - supervisor_topology: SupervisorTopology, + sidecar_process_binary_aware_network_policy: bool, #[arg(long, env = "OPENSHELL_ENABLE_USER_NAMESPACES")] enable_user_namespaces: bool, @@ -130,7 +143,11 @@ async fn main() -> Result<()> { .unwrap_or_else(openshell_core::config::default_supervisor_image), supervisor_image_pull_policy: args.supervisor_image_pull_policy.unwrap_or_default(), supervisor_sideload_method: args.supervisor_sideload_method, - supervisor_topology: args.supervisor_topology, + topology: args.topology, + sidecar: KubernetesSidecarConfig { + proxy_uid: args.sidecar_proxy_uid, + process_binary_aware_network_policy: args.sidecar_process_binary_aware_network_policy, + }, grpc_endpoint: args.grpc_endpoint.unwrap_or_default(), ssh_socket_path: args.sandbox_ssh_socket_path, client_tls_secret_name: args.client_tls_secret_name.unwrap_or_default(), diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 54501cb8c9..c6a619f3d1 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -130,8 +130,8 @@ sequenceDiagram C->>C: entrypoint: /opt/openshell/bin/openshell-sandbox ``` -The supervisor image from `deploy/docker/Dockerfile.supervisor` copies the static -`openshell-sandbox` binary to `/openshell-sandbox`. +The supervisor image from `deploy/docker/Dockerfile.supervisor` provides the +static `openshell-sandbox` binary at `/openshell-sandbox`. Mounting that image at `/opt/openshell/bin` makes the binary available as `/opt/openshell/bin/openshell-sandbox`. diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 5eb9a5c873..1401f9e7c6 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -888,9 +888,8 @@ pub fn build_container_spec_with_token_and_gpu_devices( // Side-load the supervisor binary from a standalone OCI image. // Podman resolves image_volumes at the libpod layer, mounting the // image's filesystem at the destination path without starting a - // container from it. The supervisor image is FROM scratch with just - // the binary at /openshell-sandbox, so it appears at - // /opt/openshell/bin/openshell-sandbox. + // container from it. The supervisor image exposes the binary at + // /openshell-sandbox, so it appears at /opt/openshell/bin/openshell-sandbox. image_volumes, hostname: format!("sandbox-{}", sandbox.name), // Override the image's ENTRYPOINT so the supervisor binary runs diff --git a/crates/openshell-sandbox/Cargo.toml b/crates/openshell-sandbox/Cargo.toml index 086dbe02c3..0d7ff33925 100644 --- a/crates/openshell-sandbox/Cargo.toml +++ b/crates/openshell-sandbox/Cargo.toml @@ -33,11 +33,16 @@ clap = { workspace = true } # Error handling miette = { workspace = true } +# Unix ownership for Kubernetes sidecar init setup +nix = { workspace = true } + # TLS crypto provider install (main.rs) rustls = { workspace = true } # Serialization (serde_json::json! for OCSF unmapped fields) +serde = { workspace = true } serde_json = { workspace = true } +prost = { workspace = true } # Logging tracing = { workspace = true } diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index cf193c63ce..7a1085f1bf 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -12,8 +12,9 @@ mod google_cloud_metadata; mod mechanistic_mapper; #[cfg_attr(not(target_os = "linux"), allow(dead_code))] mod metadata_server; +mod sidecar_control; -use miette::Result; +use miette::{IntoDiagnostic, Result, WrapErr}; use std::future::Future; use std::sync::Arc; use std::sync::atomic::AtomicU32; @@ -64,12 +65,20 @@ use openshell_core::denial::DenialEvent; use openshell_core::policy::{NetworkMode, NetworkPolicy, ProxyPolicy, SandboxPolicy}; use openshell_core::provider_credentials::ProviderCredentialState; use openshell_supervisor_network::opa::OpaEngine; +use openshell_supervisor_process::process::ProcessEnforcementMode; pub use openshell_supervisor_process::process::{ProcessHandle, ProcessStatus}; use openshell_supervisor_process::skills; use tokio::sync::mpsc::UnboundedSender; -#[cfg(target_os = "linux")] +#[cfg(any(test, target_os = "linux"))] use tokio::time::timeout; +const SIDECAR_NETWORK_ENFORCEMENT_MODE: &str = "sidecar-nftables"; +const SIDECAR_TLS_DIR: &str = "/etc/openshell-tls/proxy"; +const SIDECAR_CA_CERT: &str = "openshell-ca.pem"; +const SIDECAR_CA_BUNDLE: &str = "ca-bundle.pem"; +const SIDECAR_PROCESS_PROXY_ADDR: &str = "127.0.0.1:3128"; +const SIDECAR_READY_TIMEOUT_SECS: u64 = 120; + /// Run a command in the sandbox. /// /// # Errors @@ -125,17 +134,45 @@ pub async fn run_sandbox( } } + let sidecar_network_enforcement = sidecar_network_enforcement_enabled(); + let process_enforcement_mode = process_enforcement_mode(); + let process_uses_sidecar_control = + process_enabled && !network_enabled && sidecar_network_enforcement; + let mut process_control_connection = None; + let sidecar_bootstrap = if process_uses_sidecar_control { + let socket = sidecar_control_socket().ok_or_else(|| { + miette::miette!( + "{} is required for process-only sidecar topology", + openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET + ) + })?; + let (bootstrap, connection) = sidecar_control::connect_process_client( + &socket, + Duration::from_secs(SIDECAR_READY_TIMEOUT_SECS), + ) + .await?; + process_control_connection = Some(connection); + Some(bootstrap) + } else { + None + }; + // Load policy and initialize OPA engine let openshell_endpoint_for_proxy = openshell_endpoint.clone(); let sandbox_name_for_agg = sandbox.clone(); - let (mut policy, opa_engine, retained_proto, loaded_policy_origin) = load_policy( - sandbox_id.clone(), - sandbox, - openshell_endpoint.clone(), - policy_rules, - policy_data, - ) - .await?; + let (mut policy, opa_engine, retained_proto, loaded_policy_origin) = + if let Some(bootstrap) = sidecar_bootstrap.as_ref() { + load_policy_from_sidecar_bootstrap(bootstrap)? + } else { + load_policy( + sandbox_id.clone(), + sandbox, + openshell_endpoint.clone(), + policy_rules, + policy_data, + ) + .await? + }; // Override the policy's process identity with the driver-resolved UID/GID // from the pod environment. The policy defaults to the name "sandbox" which @@ -170,71 +207,89 @@ pub async fn run_sandbox( policy.process.run_as_group = Some(gid); } - // Fetch provider environment variables from the server. - // This is done after loading the policy so the sandbox can still start - // even if provider env fetch fails (graceful degradation). - let ( - provider_env_revision, - provider_env, - provider_credential_expires_at_ms, - dynamic_credentials, - ) = if let (Some(id), Some(endpoint)) = (&sandbox_id, &openshell_endpoint) { - match openshell_core::grpc_client::fetch_provider_environment(endpoint, id).await { - Ok(result) => { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .message(format!( - "Fetched provider environment [env_count:{}]", - result.environment.len() - )) - .build() - ); - ( - result.provider_env_revision, - result.environment, - result.credential_expires_at_ms, - result.dynamic_credentials, - ) - } - Err(e) => { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Other, "degraded") - .message(format!( - "Failed to fetch provider environment, continuing without: {e}" - )) - .build() - ); + #[cfg_attr(not(target_os = "linux"), allow(unused_mut))] + let (provider_credentials, mut provider_env) = + if let Some(bootstrap) = sidecar_bootstrap.as_ref() { + let provider_credentials = ProviderCredentialState::from_child_env_snapshot( + bootstrap.provider_env_revision, + bootstrap.provider_child_env.clone(), + ); + (provider_credentials, bootstrap.provider_child_env.clone()) + } else { + // Fetch provider environment variables from the server. + // This is done after loading the policy so the sandbox can still start + // even if provider env fetch fails (graceful degradation). + let ( + provider_env_revision, + provider_env, + provider_credential_expires_at_ms, + dynamic_credentials, + ) = if let (Some(id), Some(endpoint)) = (&sandbox_id, &openshell_endpoint) { + match openshell_core::grpc_client::fetch_provider_environment(endpoint, id).await { + Ok(result) => { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .message(format!( + "Fetched provider environment [env_count:{}]", + result.environment.len() + )) + .build() + ); + ( + result.provider_env_revision, + result.environment, + result.credential_expires_at_ms, + result.dynamic_credentials, + ) + } + Err(e) => { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .state(StateId::Other, "degraded") + .message(format!( + "Failed to fetch provider environment, continuing without: {e}" + )) + .build() + ); + ( + 0, + std::collections::HashMap::new(), + std::collections::HashMap::new(), + std::collections::HashMap::new(), + ) + } + } + } else { ( 0, std::collections::HashMap::new(), std::collections::HashMap::new(), std::collections::HashMap::new(), ) - } - } - } else { - ( - 0, - std::collections::HashMap::new(), - std::collections::HashMap::new(), - std::collections::HashMap::new(), - ) - }; + }; - let provider_credentials = ProviderCredentialState::from_environment( - provider_env_revision, - provider_env, - provider_credential_expires_at_ms, - dynamic_credentials, - ); - #[cfg_attr(not(target_os = "linux"), allow(unused_mut))] - let mut provider_env = provider_credentials.child_env_with_gcp_resolved(); + let provider_credentials = ProviderCredentialState::from_environment( + provider_env_revision, + provider_env, + provider_credential_expires_at_ms, + dynamic_credentials, + ); + let provider_env = provider_credentials.child_env_with_gcp_resolved(); + (provider_credentials, provider_env) + }; + let process_control_writer = process_control_connection + .as_ref() + .map(|connection| connection.writer.clone()); + let mut process_control_closed = None; + if let Some(connection) = process_control_connection { + process_control_closed = Some(connection.closed); + spawn_sidecar_control_update_watcher(connection.updates, provider_credentials.clone()); + } // Initialize the agent-proposals feature flag. Default false until the // initial settings fetch (or the poll loop) tells us otherwise. The flag @@ -258,7 +313,7 @@ pub async fn run_sandbox( // it via setns(). The RAII handle lives in this frame for the duration // of the sandbox. #[cfg(target_os = "linux")] - let netns = if network_enabled { + let netns = if network_enabled && !sidecar_network_enforcement { openshell_supervisor_process::netns::create_netns_for_proxy(&policy)? } else { None @@ -328,6 +383,80 @@ pub async fn run_sandbox( None }; + #[cfg(target_os = "linux")] + let sidecar_control_server = if network_enabled && sidecar_network_enforcement { + if !matches!(policy.network.mode, NetworkMode::Proxy) { + return Err(miette::miette!( + "sidecar network enforcement requires proxy network mode" + )); + } + let socket = sidecar_control_socket().ok_or_else(|| { + miette::miette!( + "{} is required for sidecar topology", + openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET + ) + })?; + let proto = retained_proto.as_ref().ok_or_else(|| { + miette::miette!( + "sidecar topology requires gateway policy data for the process supervisor" + ) + })?; + let ca_paths = networking.as_ref().and_then(|n| n.ca_file_paths.clone()); + Some(sidecar_control::spawn_server( + &socket, + sidecar_control::BootstrapData { + policy_proto: proto.clone(), + provider_env_revision: provider_credentials.snapshot().revision, + provider_child_env: provider_env.clone(), + proxy_ca_cert_path: ca_paths.as_ref().map(|paths| paths.0.clone()), + proxy_ca_bundle_path: ca_paths.as_ref().map(|paths| paths.1.clone()), + }, + sidecar_expected_peer()?, + )?) + } else { + None + }; + #[cfg(not(target_os = "linux"))] + let sidecar_control_server: Option = None; + + let sidecar_control_publisher = sidecar_control_server + .as_ref() + .map(sidecar_control::ServerHandle::publisher); + + #[cfg(target_os = "linux")] + let mut sidecar_control_task = None; + + #[cfg(target_os = "linux")] + if network_enabled + && sidecar_network_enforcement + && let Some(server) = sidecar_control_server + { + let trusted_ssh_socket_path = ssh_socket_path.clone().ok_or_else(|| { + miette::miette!( + "{} is required for sidecar network topology", + openshell_core::sandbox_env::SSH_SOCKET_PATH + ) + })?; + let (entrypoint_rx, connection_task) = server.into_runtime_parts(); + sidecar_control_task = Some(connection_task); + spawn_sidecar_entrypoint_handler( + entrypoint_rx, + entrypoint_pid.clone(), + opa_engine.clone(), + retained_proto.clone(), + openshell_endpoint.clone(), + sandbox_id.clone(), + std::path::PathBuf::from(trusted_ssh_socket_path), + ); + } + + #[cfg(not(target_os = "linux"))] + if network_enabled && sidecar_network_enforcement { + return Err(miette::miette!( + "sidecar network enforcement is only supported on Linux" + )); + } + // Spawn the denial-aggregator flush task. The aggregator drains denial // events from the proxy + bypass monitor, batches them, and ships // summaries to the gateway via `SubmitPolicyAnalysis`. @@ -398,11 +527,13 @@ pub async fn run_sandbox( } // Spawn background policy poll task (gRPC mode only). - if let (Some(id), Some(endpoint), Some(engine)) = ( - sandbox_id.as_deref(), - openshell_endpoint.as_deref(), - opa_engine.as_ref(), - ) { + if !process_uses_sidecar_control + && let (Some(id), Some(endpoint), Some(engine)) = ( + sandbox_id.as_deref(), + openshell_endpoint.as_deref(), + opa_engine.as_ref(), + ) + { let poll_id = id.to_string(); let poll_endpoint = endpoint.to_string(); let poll_engine = engine.clone(); @@ -424,6 +555,7 @@ pub async fn run_sandbox( ocsf_enabled: poll_ocsf_enabled, provider_credentials: poll_provider_credentials, policy_local_ctx: poll_policy_local, + sidecar_control_publisher: sidecar_control_publisher.clone(), }; tokio::spawn(async move { @@ -479,10 +611,51 @@ pub async fn run_sandbox( } } + let process_policy = process_policy_for_topology(&policy, sidecar_network_enforcement)?; + let sidecar_bootstrap_ca_file_paths = sidecar_bootstrap.as_ref().and_then(|bootstrap| { + bootstrap + .proxy_ca_cert_path + .clone() + .zip(bootstrap.proxy_ca_bundle_path.clone()) + }); + let exit_code = if process_enabled { - let ca_file_paths = networking.as_ref().and_then(|n| n.ca_file_paths.clone()); + let ca_file_paths = networking + .as_ref() + .and_then(|n| n.ca_file_paths.clone()) + .or_else(|| { + if sidecar_network_enforcement { + sidecar_bootstrap_ca_file_paths + .clone() + .or_else(sidecar_ca_file_paths) + } else { + None + } + }); + + let entrypoint_started_tx = + if process_uses_sidecar_control && let Some(writer) = process_control_writer.clone() { + let (tx, rx) = tokio::sync::oneshot::channel(); + tokio::spawn(async move { + match rx.await { + Ok(pid) => { + if let Err(err) = + sidecar_control::send_entrypoint_started(&writer, pid).await + { + warn!(error = %err, "Failed to send sidecar entrypoint event"); + } + } + Err(_closed) => { + debug!("Entrypoint exited before sidecar entrypoint event was sent"); + } + } + }); + Some(tx) + } else { + None + }; - openshell_supervisor_process::run::run_process( + let process = openshell_supervisor_process::run::run_process( program, args, workdir.as_deref(), @@ -491,8 +664,11 @@ pub async fn run_sandbox( sandbox_id.as_deref(), openshell_endpoint.as_deref(), ssh_socket_path, - &policy, + sidecar_network_enforcement, + &process_policy, + process_enforcement_mode, entrypoint_pid, + entrypoint_started_tx, provider_credentials, provider_env, ca_file_paths, @@ -502,14 +678,54 @@ pub async fn run_sandbox( bypass_denial_tx, #[cfg(target_os = "linux")] bypass_activity_tx, - ) - .await? + ); + + if let Some(control_closed) = process_control_closed.as_mut() { + tokio::select! { + result = process => result?, + _ = control_closed => { + ocsf_emit!( + AppLifecycleBuilder::new(ocsf_ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::High) + .status(StatusId::Failure) + .message( + "Authoritative network-sidecar control channel closed; terminating process container" + ) + .build() + ); + return Err(miette::miette!( + "authoritative network-sidecar control channel closed" + )); + } + } + } else { + process.await? + } } else { // Network-only sidecar mode: keep the proxy and its background - // tasks alive (held via the `networking` value) until SIGINT or - // SIGTERM. Exit 0 on clean shutdown. - wait_for_shutdown_signal().await; - 0 + // tasks alive (held via the `networking` value) until shutdown. If the + // sole authenticated process-supervisor control connection closes, + // exit non-zero so Kubernetes restarts the network sidecar and creates + // a fresh one-client bootstrap listener for the restarted agent. + #[cfg(target_os = "linux")] + if let Some(control_task) = sidecar_control_task { + tokio::select! { + () = wait_for_shutdown_signal() => 0, + result = control_task => { + warn!(?result, "Authoritative sidecar control channel exited; restarting sidecar"); + 1 + } + } + } else { + wait_for_shutdown_signal().await; + 0 + } + #[cfg(not(target_os = "linux"))] + { + wait_for_shutdown_signal().await; + 0 + } }; // Drop networking explicitly so the proxy + bypass monitor RAII @@ -552,6 +768,206 @@ async fn wait_for_shutdown_signal() { } } +fn sidecar_network_enforcement_enabled() -> bool { + std::env::var(openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE) + .is_ok_and(|value| value == SIDECAR_NETWORK_ENFORCEMENT_MODE) +} + +fn process_enforcement_mode() -> ProcessEnforcementMode { + match std::env::var(openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY) + .ok() + .as_deref() + { + Some("sidecar") => ProcessEnforcementMode::NetworkOnly, + _ => ProcessEnforcementMode::Full, + } +} + +fn sidecar_control_socket() -> Option { + std::env::var(openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET) + .ok() + .filter(|path| !path.is_empty()) + .map(std::path::PathBuf::from) +} + +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] +fn sidecar_expected_peer() -> Result { + fn required_numeric_env(name: &str) -> Result { + let value = std::env::var(name) + .into_diagnostic() + .wrap_err_with(|| format!("{name} is required for sidecar control authentication"))?; + value.parse::().into_diagnostic().wrap_err_with(|| { + format!("{name} must be a numeric ID for sidecar control authentication") + }) + } + + Ok(sidecar_control::ExpectedPeer { + uid: required_numeric_env(openshell_core::sandbox_env::SANDBOX_UID)?, + gid: required_numeric_env(openshell_core::sandbox_env::SANDBOX_GID)?, + }) +} + +type LoadedPolicyBundle = ( + SandboxPolicy, + Option>, + Option, + LoadedPolicyOrigin, +); + +fn load_policy_from_sidecar_bootstrap( + bootstrap: &sidecar_control::BootstrapData, +) -> Result { + let proto = bootstrap.policy_proto.clone(); + let opa_engine = Some(Arc::new(OpaEngine::from_proto(&proto)?)); + let policy = SandboxPolicy::try_from(proto.clone())?; + info!("Loaded sidecar policy from control socket bootstrap"); + Ok(( + policy, + opa_engine, + Some(proto), + LoadedPolicyOrigin::Gateway { revision: None }, + )) +} + +fn spawn_sidecar_control_update_watcher( + mut updates: tokio::sync::mpsc::UnboundedReceiver, + provider_credentials: ProviderCredentialState, +) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + while let Some(update) = updates.recv().await { + match update { + sidecar_control::ControlUpdate::ProviderEnvUpdated { + revision, + provider_child_env, + } => { + if revision <= provider_credentials.snapshot().revision { + continue; + } + let env_count = provider_credentials + .install_child_env_snapshot(revision, provider_child_env); + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped("provider_env_revision", serde_json::json!(revision)) + .message(format!( + "Sidecar provider environment refreshed [revision:{revision} env_count:{env_count}]" + )) + .build() + ); + } + sidecar_control::ControlUpdate::PolicyUpdated { + policy_proto, + policy_hash, + config_revision, + } => { + debug!( + version = policy_proto.version, + policy_hash, + config_revision, + "Received sidecar policy update for process supervisor" + ); + } + } + } + }) +} + +#[cfg(target_os = "linux")] +fn spawn_sidecar_entrypoint_handler( + mut entrypoint_rx: tokio::sync::mpsc::Receiver, + entrypoint_pid: Arc, + opa_engine: Option>, + retained_proto: Option, + openshell_endpoint: Option, + sandbox_id: Option, + trusted_ssh_socket_path: std::path::PathBuf, +) { + tokio::spawn(async move { + let mut session_started = false; + let mut trusted_supervisor_pid = None; + while let Some(started) = entrypoint_rx.recv().await { + entrypoint_pid.store(started.pid, std::sync::atomic::Ordering::Release); + if started.start_session { + info!( + pid = started.pid, + ssh_socket = %trusted_ssh_socket_path.display(), + "Sidecar process supervisor reported entrypoint start" + ); + } else { + trusted_supervisor_pid = Some(started.pid); + info!( + pid = started.pid, + "Sidecar process supervisor reported initial process anchor" + ); + } + + if let (Some(engine), Some(proto)) = (opa_engine.as_ref(), retained_proto.as_ref()) { + match engine.reload_from_proto_with_pid(proto, started.pid) { + Ok(()) => info!( + pid = started.pid, + "Policy binary symlink resolution complete for sidecar process anchor" + ), + Err(err) => warn!( + error = %err, + pid = started.pid, + "Failed to rebuild OPA engine with sidecar process anchor PID" + ), + } + } + + if started.start_session + && !session_started + && let (Some(endpoint), Some(id)) = + (openshell_endpoint.as_ref(), sandbox_id.as_ref()) + { + let Some(supervisor_pid) = trusted_supervisor_pid else { + warn!( + pid = started.pid, + "Ignoring sidecar entrypoint event before authenticated supervisor anchor" + ); + continue; + }; + openshell_supervisor_process::supervisor_session::spawn( + endpoint.clone(), + id.clone(), + trusted_ssh_socket_path.clone(), + None, + Some(supervisor_pid), + ); + session_started = true; + info!("sidecar supervisor session task spawned"); + } + } + }); +} + +fn sidecar_ca_file_paths() -> Option<(std::path::PathBuf, std::path::PathBuf)> { + let tls_dir = std::env::var(openshell_core::sandbox_env::PROXY_TLS_DIR) + .unwrap_or_else(|_| SIDECAR_TLS_DIR.to_string()); + let cert = std::path::Path::new(&tls_dir).join(SIDECAR_CA_CERT); + let bundle = std::path::Path::new(&tls_dir).join(SIDECAR_CA_BUNDLE); + (cert.exists() && bundle.exists()).then_some((cert, bundle)) +} + +fn process_policy_for_topology( + policy: &SandboxPolicy, + sidecar_network_enforcement: bool, +) -> Result { + let mut process_policy = policy.clone(); + if sidecar_network_enforcement && matches!(process_policy.network.mode, NetworkMode::Proxy) { + let proxy = process_policy + .network + .proxy + .get_or_insert(ProxyPolicy { http_addr: None }); + if proxy.http_addr.is_none() { + proxy.http_addr = Some(SIDECAR_PROCESS_PROXY_ADDR.parse().into_diagnostic()?); + } + } + Ok(process_policy) +} + /// Flush aggregated denial summaries to the gateway via `SubmitPolicyAnalysis`. async fn flush_proposals_to_gateway( endpoint: &str, @@ -1947,6 +2363,7 @@ struct PolicyPollLoopContext { ocsf_enabled: Arc, provider_credentials: ProviderCredentialState, policy_local_ctx: Option>, + sidecar_control_publisher: Option, } async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { @@ -2058,12 +2475,20 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { .await { Ok(env_result) => { - let env_count = ctx.provider_credentials.install_environment( + ctx.provider_credentials.install_environment( env_result.provider_env_revision, env_result.environment, env_result.credential_expires_at_ms, env_result.dynamic_credentials, ); + let child_env = ctx.provider_credentials.child_env_with_gcp_resolved(); + let env_count = child_env.len(); + if let Some(publisher) = ctx.sidecar_control_publisher.as_ref() { + publisher.publish_provider_env( + env_result.provider_env_revision, + child_env.clone(), + ); + } current_provider_env_revision = env_result.provider_env_revision; ocsf_emit!( ConfigStateChangeBuilder::new(ocsf_ctx()) @@ -2072,10 +2497,11 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { .state(StateId::Enabled, "loaded") .unmapped( "provider_env_revision", - serde_json::json!(current_provider_env_revision) + serde_json::json!(env_result.provider_env_revision) ) .message(format!( - "Provider environment refreshed [revision:{current_provider_env_revision} env_count:{env_count}]" + "Provider environment refreshed [revision:{} env_count:{env_count}]", + env_result.provider_env_revision )) .build() ); @@ -2111,6 +2537,13 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { if let Some(policy_local_ctx) = ctx.policy_local_ctx.as_ref() { policy_local_ctx.set_current_policy(policy.clone()).await; } + if let Some(publisher) = ctx.sidecar_control_publisher.as_ref() { + publisher.publish_policy( + policy.clone(), + result.policy_hash.clone(), + result.config_revision, + ); + } if result.global_policy_version > 0 { ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) .severity(SeverityId::Informational) @@ -2317,8 +2750,24 @@ fn format_setting_value(es: &openshell_core::proto::EffectiveSetting) -> String )] mod tests { use super::*; + use openshell_core::policy::{ + FilesystemPolicy, LandlockPolicy, NetworkMode, NetworkPolicy, ProcessPolicy, ProxyPolicy, + }; use std::sync::atomic::{AtomicBool, Ordering}; + fn proxy_policy(http_addr: Option) -> SandboxPolicy { + SandboxPolicy { + version: 1, + filesystem: FilesystemPolicy::default(), + network: NetworkPolicy { + mode: NetworkMode::Proxy, + proxy: Some(ProxyPolicy { http_addr }), + }, + landlock: LandlockPolicy::default(), + process: ProcessPolicy::default(), + } + } + fn effective_bool(value: bool) -> openshell_core::proto::EffectiveSetting { openshell_core::proto::EffectiveSetting { value: Some(openshell_core::proto::SettingValue { @@ -2330,6 +2779,100 @@ mod tests { } } + #[test] + fn sidecar_process_policy_sets_loopback_proxy_addr() { + let policy = proxy_policy(None); + + let process_policy = process_policy_for_topology(&policy, true).unwrap(); + + let http_addr = process_policy + .network + .proxy + .and_then(|proxy| proxy.http_addr) + .expect("sidecar process policy should set proxy address"); + assert_eq!(http_addr.to_string(), SIDECAR_PROCESS_PROXY_ADDR); + assert!( + policy + .network + .proxy + .as_ref() + .expect("original policy should keep proxy config") + .http_addr + .is_none(), + "process policy normalization must not mutate the network policy" + ); + } + + #[test] + fn non_sidecar_process_policy_preserves_proxy_addr() { + let policy = proxy_policy(None); + + let process_policy = process_policy_for_topology(&policy, false).unwrap(); + + assert!( + process_policy + .network + .proxy + .and_then(|proxy| proxy.http_addr) + .is_none() + ); + } + + #[tokio::test] + async fn sidecar_control_provider_env_update_installs_newer_revision() { + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + let provider_credentials = ProviderCredentialState::from_child_env_snapshot( + 1, + std::collections::HashMap::from([("TOKEN".to_string(), "old".to_string())]), + ); + let handle = spawn_sidecar_control_update_watcher(rx, provider_credentials.clone()); + + tx.send(sidecar_control::ControlUpdate::ProviderEnvUpdated { + revision: 2, + provider_child_env: std::collections::HashMap::from([( + "TOKEN".to_string(), + "new".to_string(), + )]), + }) + .unwrap(); + + timeout(Duration::from_secs(1), async { + loop { + if provider_credentials.snapshot().revision == 2 { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .unwrap(); + let snapshot = provider_credentials.snapshot(); + assert_eq!(snapshot.revision, 2); + assert_eq!( + snapshot.child_env.get("TOKEN").map(String::as_str), + Some("new") + ); + + tx.send(sidecar_control::ControlUpdate::ProviderEnvUpdated { + revision: 1, + provider_child_env: std::collections::HashMap::from([( + "TOKEN".to_string(), + "stale".to_string(), + )]), + }) + .unwrap(); + tokio::time::sleep(Duration::from_millis(20)).await; + assert_eq!( + provider_credentials + .snapshot() + .child_env + .get("TOKEN") + .map(String::as_str), + Some("new") + ); + handle.abort(); + } + #[test] fn apply_ocsf_json_setting_enables_from_initial_settings_snapshot() { let enabled = AtomicBool::new(false); diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index 91b145c2e0..71f881f68a 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -35,15 +35,36 @@ const DEBUG_RPC_SUBCOMMAND: &str = "debug-rpc"; /// Default `--mode` value: run both supervisor leaves in a single binary. const DEFAULT_MODE: &str = "network,process"; +const SIDECAR_STATE_DIR: &str = "/run/openshell-sidecar"; +const SIDECAR_TLS_DIR: &str = "/etc/openshell-tls/proxy"; +#[cfg(target_os = "linux")] +const CLIENT_TLS_DIR: &str = "/etc/openshell-tls/client"; +#[cfg(target_os = "linux")] +const SIDECAR_CLIENT_TLS_SUBDIR: &str = "client"; +#[cfg(target_os = "linux")] +const CLIENT_TLS_FILES: [&str; 3] = ["ca.crt", "tls.crt", "tls.key"]; +#[cfg(target_os = "linux")] +const SIDECAR_STATE_DIR_MODE: u32 = 0o2775; +#[cfg(target_os = "linux")] +const SIDECAR_TLS_DIR_MODE: u32 = 0o755; +#[cfg(target_os = "linux")] +const SIDECAR_TLS_STAGING_DIR_MODE: u32 = 0o700; +#[cfg(target_os = "linux")] +const SIDECAR_CLIENT_TLS_DIR_MODE: u32 = 0o750; +#[cfg(target_os = "linux")] +const SIDECAR_CLIENT_TLS_FILE_MODE: u32 = 0o400; /// Which supervisor leaves are enabled in this process. /// /// Parsed from a comma-separated `--mode` value, e.g. `network`, -/// `process`, or `network,process`. At least one must be set. +/// `process`, or `network,process`. `network-init` is a one-shot setup mode +/// used by the Kubernetes sidecar topology and cannot be combined with other +/// mode components. At least one must be set. #[derive(Clone, Copy, Debug)] struct Mode { network: bool, process: bool, + network_init: bool, } impl std::str::FromStr for Mode { @@ -53,20 +74,27 @@ impl std::str::FromStr for Mode { let mut mode = Self { network: false, process: false, + network_init: false, }; for part in s.split(',').map(str::trim).filter(|p| !p.is_empty()) { match part { "network" => mode.network = true, "process" => mode.process = true, + "network-init" => mode.network_init = true, other => { return Err(format!( - "unknown mode component '{other}' (expected 'network' and/or 'process')" + "unknown mode component '{other}' (expected 'network', 'process', or 'network-init')" )); } } } - if !mode.network && !mode.process { - return Err("--mode must enable at least one of: network, process".into()); + if mode.network_init && (mode.network || mode.process) { + return Err("--mode=network-init cannot be combined with other components".into()); + } + if !mode.network && !mode.process && !mode.network_init { + return Err( + "--mode must enable at least one of: network, process, network-init".into(), + ); } Ok(mode) } @@ -125,7 +153,8 @@ struct Args { #[arg(long, default_value = "warn", env = openshell_core::sandbox_env::LOG_LEVEL)] log_level: String, - /// Filesystem path to the Unix socket the embedded SSH daemon binds. + /// Unix socket the embedded SSH daemon binds. On Linux, a value beginning + /// with `@` selects an abstract socket in the network namespace. /// The supervisor bridges `RelayStream` traffic from the gateway onto /// this socket; nothing else should connect to it. #[arg(long, env = openshell_core::sandbox_env::SSH_SOCKET_PATH)] @@ -149,9 +178,28 @@ struct Args { /// "network" and/or "process". Defaults to both (single-binary /// topology). Use --mode=network for a network-only sidecar, or /// --mode=process for a process-only supervisor when network - /// enforcement runs in another pod. + /// enforcement runs in another pod. Use --mode=network-init only in + /// the Kubernetes init container that prepares sidecar nftables. #[arg(long, default_value = DEFAULT_MODE)] mode: Mode, + + /// UID that the long-running Kubernetes network sidecar will run as. + /// `--mode=network-init` installs nftables rules that exempt this UID. + #[arg(long, env = "OPENSHELL_PROXY_UID", default_value_t = 1337)] + proxy_uid: u32, + + /// GID assigned to shared sidecar state directories. Defaults to + /// `--proxy-uid` when omitted. + #[arg(long, env = "OPENSHELL_PROXY_GID")] + proxy_gid: Option, + + /// Shared state directory between the network init container and sidecar. + #[arg(long, env = "OPENSHELL_SIDECAR_STATE_DIR", default_value = SIDECAR_STATE_DIR)] + sidecar_state_dir: String, + + /// Shared TLS work directory between the network init container and sidecar. + #[arg(long, env = "OPENSHELL_PROXY_TLS_DIR", default_value = SIDECAR_TLS_DIR)] + sidecar_tls_dir: String, } /// Copy the running executable to `dest`, creating parent directories as @@ -194,6 +242,189 @@ fn copy_self(dest: &str) -> Result<()> { Ok(()) } +#[cfg(target_os = "linux")] +fn prepare_sidecar_directory(path: &Path, uid: u32, gid: u32, mode: u32) -> Result<()> { + use miette::Context as _; + use nix::unistd::{Gid, Uid, chown}; + use std::os::unix::fs::PermissionsExt; + + std::fs::create_dir_all(path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create sidecar directory {}", path.display()))?; + let mut perms = std::fs::metadata(path).into_diagnostic()?.permissions(); + perms.set_mode(mode); + std::fs::set_permissions(path, perms) + .into_diagnostic() + .wrap_err_with(|| format!("failed to chmod sidecar directory {}", path.display()))?; + chown(path, Some(Uid::from_raw(uid)), Some(Gid::from_raw(gid))) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to chown sidecar directory {} to {uid}:{gid}", + path.display() + ) + })?; + Ok(()) +} + +#[cfg(target_os = "linux")] +fn prepare_sidecar_directory_for_current_user(path: &Path, mode: u32) -> Result<()> { + use miette::Context as _; + use nix::unistd::{Gid, Uid, chown}; + use std::os::unix::fs::PermissionsExt; + + let uid = Uid::current(); + let gid = Gid::current(); + std::fs::create_dir_all(path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create sidecar directory {}", path.display()))?; + chown(path, Some(uid), Some(gid)) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to chown sidecar directory {} to {}:{}", + path.display(), + uid.as_raw(), + gid.as_raw() + ) + })?; + let mut perms = std::fs::metadata(path).into_diagnostic()?.permissions(); + perms.set_mode(mode); + std::fs::set_permissions(path, perms) + .into_diagnostic() + .wrap_err_with(|| format!("failed to chmod sidecar directory {}", path.display()))?; + Ok(()) +} + +#[cfg(target_os = "linux")] +fn copy_sidecar_client_tls_if_present( + source_dir: &Path, + sidecar_tls_dir: &Path, + uid: u32, + gid: u32, +) -> Result<()> { + use miette::Context as _; + use nix::unistd::{Gid, Uid, chown}; + use std::os::unix::fs::PermissionsExt; + + if !source_dir.exists() { + return Ok(()); + } + + let dest_dir = sidecar_tls_dir.join(SIDECAR_CLIENT_TLS_SUBDIR); + prepare_sidecar_directory_for_current_user(&dest_dir, SIDECAR_TLS_STAGING_DIR_MODE)?; + for file_name in CLIENT_TLS_FILES { + let source = source_dir.join(file_name); + if !source.exists() { + return Err(miette::miette!( + "client TLS source file is missing: {}", + source.display() + )); + } + let dest = dest_dir.join(file_name); + if dest.exists() { + std::fs::remove_file(&dest) + .into_diagnostic() + .wrap_err_with(|| { + format!("failed to remove stale client TLS file {}", dest.display()) + })?; + } + std::fs::copy(&source, &dest) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to copy client TLS file {} to {}", + source.display(), + dest.display() + ) + })?; + let mut perms = std::fs::metadata(&dest).into_diagnostic()?.permissions(); + perms.set_mode(SIDECAR_CLIENT_TLS_FILE_MODE); + std::fs::set_permissions(&dest, perms) + .into_diagnostic() + .wrap_err_with(|| { + format!("failed to chmod copied client TLS file {}", dest.display()) + })?; + chown(&dest, Some(Uid::from_raw(uid)), Some(Gid::from_raw(gid))) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to chown copied client TLS file {} to {uid}:{gid}", + dest.display() + ) + })?; + } + + prepare_sidecar_directory(&dest_dir, uid, gid, SIDECAR_CLIENT_TLS_DIR_MODE)?; + + Ok(()) +} + +#[cfg(target_os = "linux")] +fn run_network_init( + proxy_user_id: u32, + proxy_primary_group_id: u32, + sidecar_state_dir: &str, + sidecar_tls_dir: &str, +) -> Result<()> { + validate_network_init_ids(proxy_user_id, proxy_primary_group_id)?; + + let sidecar_state_dir = Path::new(sidecar_state_dir); + let sidecar_tls_dir = Path::new(sidecar_tls_dir); + prepare_sidecar_directory( + sidecar_state_dir, + proxy_user_id, + proxy_primary_group_id, + SIDECAR_STATE_DIR_MODE, + )?; + // The init container runs as uid 0 with CAP_DAC_OVERRIDE dropped. Keep the + // TLS work directory owned by the init user until the client cert copy is + // complete, then hand it to the long-running proxy UID. + prepare_sidecar_directory_for_current_user(sidecar_tls_dir, SIDECAR_TLS_DIR_MODE)?; + copy_sidecar_client_tls_if_present( + Path::new(CLIENT_TLS_DIR), + sidecar_tls_dir, + proxy_user_id, + proxy_primary_group_id, + )?; + prepare_sidecar_directory( + sidecar_tls_dir, + proxy_user_id, + proxy_primary_group_id, + SIDECAR_TLS_DIR_MODE, + )?; + openshell_supervisor_process::netns::install_sidecar_bypass_rules(proxy_user_id) +} + +#[cfg(target_os = "linux")] +fn validate_network_init_ids(proxy_user_id: u32, proxy_primary_group_id: u32) -> Result<()> { + if proxy_user_id != 0 && proxy_user_id < openshell_policy::MIN_SANDBOX_UID { + return Err(miette::miette!( + "--proxy-uid must be 0 or at least {}", + openshell_policy::MIN_SANDBOX_UID + )); + } + if proxy_primary_group_id < openshell_policy::MIN_SANDBOX_UID { + return Err(miette::miette!( + "--proxy-gid must be at least {}", + openshell_policy::MIN_SANDBOX_UID + )); + } + Ok(()) +} + +#[cfg(not(target_os = "linux"))] +fn run_network_init( + _proxy_uid: u32, + _proxy_gid: u32, + _sidecar_state_dir: &str, + _sidecar_tls_dir: &str, +) -> Result<()> { + Err(miette::miette!( + "--mode=network-init is only supported on Linux" + )) +} + fn main() -> Result<()> { // Handle `copy-self ` before clap so it works without any of the // sandbox flags. Kubernetes init containers invoke this path to seed an @@ -222,6 +453,16 @@ fn main() -> Result<()> { let args = Args::parse(); + if args.mode.network_init { + let proxy_gid = args.proxy_gid.unwrap_or(args.proxy_uid); + return run_network_init( + args.proxy_uid, + proxy_gid, + &args.sidecar_state_dir, + &args.sidecar_tls_dir, + ); + } + // Try to open a rolling log file; fall back to stderr-only logging if it fails // (e.g., /var/log is not writable in custom workload images). // Rotates daily, keeps the 3 most recent files to bound disk usage. @@ -421,4 +662,50 @@ mod tests { let final_path = dest_dir.join("openshell-sandbox"); assert!(final_path.exists(), "binary should land inside dest dir"); } + + #[test] + fn mode_parses_network_init_standalone() { + let mode = "network-init".parse::().unwrap(); + assert!(mode.network_init); + assert!(!mode.network); + assert!(!mode.process); + } + + #[test] + fn mode_rejects_combined_network_init() { + let err = "network-init,network".parse::().unwrap_err(); + assert!(err.contains("cannot be combined")); + } + + #[test] + fn mode_rejects_empty_value() { + let err = "".parse::().unwrap_err(); + assert!(err.contains("at least one")); + } + + #[cfg(target_os = "linux")] + #[test] + fn sidecar_tls_modes_preserve_proxy_owned_parent_and_private_client_dir() { + assert_eq!(SIDECAR_TLS_DIR_MODE, 0o755); + assert_eq!(SIDECAR_TLS_STAGING_DIR_MODE, 0o700); + assert_eq!(SIDECAR_CLIENT_TLS_DIR_MODE, 0o750); + assert_eq!(SIDECAR_CLIENT_TLS_FILE_MODE, 0o400); + } + + #[cfg(target_os = "linux")] + #[test] + fn network_init_accepts_root_proxy_uid_for_binary_aware_sidecar() { + validate_network_init_ids(0, openshell_policy::MIN_SANDBOX_UID).unwrap(); + } + + #[cfg(target_os = "linux")] + #[test] + fn network_init_still_rejects_low_non_root_proxy_ids() { + let uid_err = + validate_network_init_ids(999, openshell_policy::MIN_SANDBOX_UID).unwrap_err(); + assert!(uid_err.to_string().contains("--proxy-uid")); + + let gid_err = validate_network_init_ids(0, 999).unwrap_err(); + assert!(gid_err.to_string().contains("--proxy-gid")); + } } diff --git a/crates/openshell-sandbox/src/sidecar_control.rs b/crates/openshell-sandbox/src/sidecar_control.rs new file mode 100644 index 0000000000..03cfeff886 --- /dev/null +++ b/crates/openshell-sandbox/src/sidecar_control.rs @@ -0,0 +1,783 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Local control channel for Kubernetes sidecar topology. +//! +//! The network sidecar owns gateway credentials. The process supervisor in the +//! agent container connects over this Unix socket to receive policy/provider +//! state without mounting gateway credentials into the agent container. + +use miette::{IntoDiagnostic, Result, WrapErr}; +use prost::Message; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, RwLock}; +use std::time::Duration; +use tokio::io::{AsyncBufReadExt, AsyncWrite, AsyncWriteExt, BufReader}; +use tokio::net::UnixListener; +use tokio::net::unix::OwnedWriteHalf; +use tokio::sync::{Mutex, broadcast, mpsc}; +use tracing::{debug, info, warn}; + +#[derive(Debug, Clone)] +pub struct BootstrapData { + pub policy_proto: openshell_core::proto::SandboxPolicy, + pub provider_env_revision: u64, + pub provider_child_env: HashMap, + pub proxy_ca_cert_path: Option, + pub proxy_ca_bundle_path: Option, +} + +#[derive(Debug, Clone)] +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] +pub struct EntrypointStarted { + pub pid: u32, + pub start_session: bool, +} + +#[derive(Debug, Clone, Copy)] +pub struct ExpectedPeer { + pub uid: u32, + pub gid: u32, +} + +#[derive(Debug, Clone)] +pub enum ControlUpdate { + ProviderEnvUpdated { + revision: u64, + provider_child_env: HashMap, + }, + PolicyUpdated { + policy_proto: openshell_core::proto::SandboxPolicy, + policy_hash: String, + config_revision: u64, + }, +} + +#[derive(Clone)] +pub struct Publisher { + state: Arc>, + updates: broadcast::Sender, +} + +impl Publisher { + pub fn publish_provider_env(&self, revision: u64, provider_child_env: HashMap) { + { + let mut state = self.state.write().expect("sidecar control state poisoned"); + if revision <= state.provider_env_revision { + return; + } + state.provider_env_revision = revision; + state.provider_child_env.clone_from(&provider_child_env); + } + + let _ = self.updates.send(WireServerMessage::ProviderEnvUpdated { + revision, + provider_child_env, + }); + } + + pub fn publish_policy( + &self, + policy_proto: openshell_core::proto::SandboxPolicy, + policy_hash: String, + config_revision: u64, + ) { + { + let mut state = self.state.write().expect("sidecar control state poisoned"); + state.policy_proto = policy_proto.clone(); + } + + let _ = self.updates.send(WireServerMessage::PolicyUpdated { + policy_proto: policy_proto.encode_to_vec(), + policy_hash, + config_revision, + }); + } +} + +pub struct ServerHandle { + publisher: Publisher, + #[cfg_attr(not(target_os = "linux"), allow(dead_code))] + entrypoint_rx: mpsc::Receiver, + connection_task: tokio::task::JoinHandle<()>, +} + +impl ServerHandle { + pub fn publisher(&self) -> Publisher { + self.publisher.clone() + } + + #[cfg(test)] + pub fn into_entrypoint_receiver(self) -> mpsc::Receiver { + self.entrypoint_rx + } + + #[cfg_attr(not(target_os = "linux"), allow(dead_code))] + pub fn into_runtime_parts( + self, + ) -> ( + mpsc::Receiver, + tokio::task::JoinHandle<()>, + ) { + (self.entrypoint_rx, self.connection_task) + } +} + +pub struct ProcessConnection { + pub writer: Arc>, + pub updates: mpsc::UnboundedReceiver, + pub closed: tokio::sync::oneshot::Receiver<()>, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +enum WireClientMessage { + BootstrapRequest { supervisor_pid: u32 }, + EntrypointStarted { pid: u32 }, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +enum WireServerMessage { + BootstrapResponse { + policy_proto: Vec, + provider_env_revision: u64, + provider_child_env: HashMap, + proxy_ca_cert_path: Option, + proxy_ca_bundle_path: Option, + }, + ProviderEnvUpdated { + revision: u64, + provider_child_env: HashMap, + }, + PolicyUpdated { + policy_proto: Vec, + policy_hash: String, + config_revision: u64, + }, +} + +impl BootstrapData { + #[cfg_attr(not(target_os = "linux"), allow(dead_code))] + fn to_wire(&self) -> WireServerMessage { + WireServerMessage::BootstrapResponse { + policy_proto: self.policy_proto.encode_to_vec(), + provider_env_revision: self.provider_env_revision, + provider_child_env: self.provider_child_env.clone(), + proxy_ca_cert_path: self + .proxy_ca_cert_path + .as_ref() + .map(|path| path.display().to_string()), + proxy_ca_bundle_path: self + .proxy_ca_bundle_path + .as_ref() + .map(|path| path.display().to_string()), + } + } +} + +impl TryFrom for BootstrapData { + type Error = miette::Report; + + fn try_from(message: WireServerMessage) -> Result { + let WireServerMessage::BootstrapResponse { + policy_proto, + provider_env_revision, + provider_child_env, + proxy_ca_cert_path, + proxy_ca_bundle_path, + } = message + else { + return Err(miette::miette!( + "expected sidecar bootstrap response, received update message" + )); + }; + + let policy_proto = openshell_core::proto::SandboxPolicy::decode(policy_proto.as_slice()) + .into_diagnostic() + .wrap_err("failed to decode sidecar bootstrap policy")?; + + Ok(Self { + policy_proto, + provider_env_revision, + provider_child_env, + proxy_ca_cert_path: proxy_ca_cert_path.map(PathBuf::from), + proxy_ca_bundle_path: proxy_ca_bundle_path.map(PathBuf::from), + }) + } +} + +impl TryFrom for ControlUpdate { + type Error = miette::Report; + + fn try_from(message: WireServerMessage) -> Result { + match message { + WireServerMessage::ProviderEnvUpdated { + revision, + provider_child_env, + } => Ok(Self::ProviderEnvUpdated { + revision, + provider_child_env, + }), + WireServerMessage::PolicyUpdated { + policy_proto, + policy_hash, + config_revision, + } => { + let policy_proto = + openshell_core::proto::SandboxPolicy::decode(policy_proto.as_slice()) + .into_diagnostic() + .wrap_err("failed to decode sidecar policy update")?; + Ok(Self::PolicyUpdated { + policy_proto, + policy_hash, + config_revision, + }) + } + WireServerMessage::BootstrapResponse { .. } => Err(miette::miette!( + "unexpected sidecar bootstrap response after initial handshake" + )), + } + } +} + +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] +pub fn spawn_server( + path: &Path, + bootstrap: BootstrapData, + expected_peer: ExpectedPeer, +) -> Result { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to create sidecar control socket dir {}", + parent.display() + ) + })?; + } + match std::fs::remove_file(path) { + Ok(()) => {} + Err(err) if err.kind() == std::io::ErrorKind::NotFound => {} + Err(err) => { + return Err(err).into_diagnostic().wrap_err_with(|| { + format!( + "failed to remove stale sidecar control socket {}", + path.display() + ) + }); + } + } + + let listener = UnixListener::bind(path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to bind sidecar control socket {}", path.display()))?; + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o660)) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to set permissions on sidecar control socket {}", + path.display() + ) + })?; + } + + let state = Arc::new(RwLock::new(bootstrap)); + let (updates, _) = broadcast::channel(32); + let (entrypoint_tx, entrypoint_rx) = mpsc::channel(8); + let publisher = Publisher { + state: state.clone(), + updates: updates.clone(), + }; + + let connection_task = tokio::spawn(accept_authoritative_connection( + listener, + path.to_path_buf(), + expected_peer, + state, + updates, + entrypoint_tx, + )); + info!(path = %path.display(), "Sidecar control socket listening"); + + Ok(ServerHandle { + publisher, + entrypoint_rx, + connection_task, + }) +} + +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] +async fn accept_authoritative_connection( + listener: UnixListener, + socket_path: PathBuf, + expected_peer: ExpectedPeer, + state: Arc>, + updates: broadcast::Sender, + entrypoint_tx: mpsc::Sender, +) { + let stream = match listener.accept().await { + Ok((stream, _addr)) => stream, + Err(err) => { + warn!(error = %err, "Failed to accept authoritative sidecar control connection"); + return; + } + }; + + // The process supervisor connects before it launches the workload. Drop + // the listener and unlink its pathname after that first accept so workload + // processes can neither open a second control channel nor impersonate a + // restarted server at the trusted path. + drop(listener); + if let Err(err) = std::fs::remove_file(&socket_path) + && err.kind() != std::io::ErrorKind::NotFound + { + warn!( + path = %socket_path.display(), + error = %err, + "Failed to unlink accepted sidecar control socket" + ); + } + + if let Err(err) = handle_connection(stream, expected_peer, state, updates, entrypoint_tx).await + { + warn!(error = %err, "Authoritative sidecar control connection closed"); + } +} + +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] +async fn handle_connection( + stream: tokio::net::UnixStream, + expected_peer: ExpectedPeer, + state: Arc>, + updates: broadcast::Sender, + entrypoint_tx: mpsc::Sender, +) -> Result<()> { + let credentials = stream + .peer_cred() + .into_diagnostic() + .wrap_err("failed to read sidecar control peer credentials")?; + if credentials.uid() != expected_peer.uid || credentials.gid() != expected_peer.gid { + return Err(miette::miette!( + "sidecar control peer identity mismatch: expected uid:gid {}:{}, got {}:{}", + expected_peer.uid, + expected_peer.gid, + credentials.uid(), + credentials.gid(), + )); + } + let peer_pid = credentials + .pid() + .and_then(|pid| u32::try_from(pid).ok()) + .ok_or_else(|| miette::miette!("sidecar control peer PID is unavailable"))?; + + let (reader, mut writer) = stream.into_split(); + let mut lines = BufReader::new(reader).lines(); + + let first_line = + lines.next_line().await.into_diagnostic()?.ok_or_else(|| { + miette::miette!("sidecar control client disconnected before bootstrap") + })?; + match decode_client_message(&first_line)? { + WireClientMessage::BootstrapRequest { supervisor_pid } => { + if supervisor_pid == 0 || supervisor_pid != peer_pid { + return Err(miette::miette!( + "sidecar bootstrap PID mismatch: peer PID {peer_pid}, claimed PID {supervisor_pid}" + )); + } + entrypoint_tx + .send(EntrypointStarted { + pid: supervisor_pid, + start_session: false, + }) + .await + .map_err(|_| miette::miette!("sidecar entrypoint receiver closed"))?; + } + WireClientMessage::EntrypointStarted { .. } => { + return Err(miette::miette!( + "sidecar control client sent entrypoint event before bootstrap" + )); + } + } + + let bootstrap = { + let state = state.read().expect("sidecar control state poisoned"); + state.to_wire() + }; + write_json_line(&mut writer, &bootstrap).await?; + + let mut update_rx = updates.subscribe(); + loop { + tokio::select! { + line = lines.next_line() => { + let Some(line) = line.into_diagnostic()? else { + return Ok(()); + }; + match decode_client_message(&line)? { + WireClientMessage::BootstrapRequest { .. } => { + debug!("Ignoring duplicate sidecar bootstrap request"); + } + WireClientMessage::EntrypointStarted { pid } => { + if pid == 0 { + warn!("Ignoring sidecar entrypoint event with pid=0"); + continue; + } + entrypoint_tx + .send(EntrypointStarted { + pid, + start_session: true, + }) + .await + .map_err(|_| miette::miette!("sidecar entrypoint receiver closed"))?; + } + } + } + update = update_rx.recv() => { + match update { + Ok(message) => write_json_line(&mut writer, &message).await?, + Err(broadcast::error::RecvError::Lagged(skipped)) => { + warn!(skipped, "Sidecar control client lagged behind updates"); + } + Err(broadcast::error::RecvError::Closed) => return Ok(()), + } + } + } + } +} + +pub async fn connect_process_client( + path: &Path, + timeout: Duration, +) -> Result<(BootstrapData, ProcessConnection)> { + let stream = connect_with_retry(path, timeout).await?; + let (reader, mut writer) = stream.into_split(); + write_json_line( + &mut writer, + &WireClientMessage::BootstrapRequest { + supervisor_pid: std::process::id(), + }, + ) + .await?; + + let mut lines = BufReader::new(reader).lines(); + let first_line = lines + .next_line() + .await + .into_diagnostic()? + .ok_or_else(|| miette::miette!("sidecar control closed before bootstrap response"))?; + let bootstrap = BootstrapData::try_from(decode_server_message(&first_line)?)?; + + let (update_tx, updates) = mpsc::unbounded_channel(); + let (closed_tx, closed) = tokio::sync::oneshot::channel(); + tokio::spawn(async move { + while let Ok(Some(line)) = lines.next_line().await { + match decode_server_message(&line).and_then(ControlUpdate::try_from) { + Ok(update) => { + if update_tx.send(update).is_err() { + break; + } + } + Err(err) => { + warn!(error = %err, "Ignoring invalid sidecar control update"); + } + } + } + let _ = closed_tx.send(()); + }); + + Ok(( + bootstrap, + ProcessConnection { + writer: Arc::new(Mutex::new(writer)), + updates, + closed, + }, + )) +} + +async fn connect_with_retry(path: &Path, timeout: Duration) -> Result { + let deadline = tokio::time::Instant::now() + timeout; + loop { + match tokio::net::UnixStream::connect(path).await { + Ok(stream) => return Ok(stream), + Err(err) if tokio::time::Instant::now() < deadline => { + debug!( + path = %path.display(), + error = %err, + "Waiting for sidecar control socket" + ); + tokio::time::sleep(Duration::from_millis(100)).await; + } + Err(err) => { + return Err(err).into_diagnostic().wrap_err_with(|| { + format!( + "timed out waiting for sidecar control socket {}", + path.display() + ) + }); + } + } + } +} + +pub async fn send_entrypoint_started(writer: &Arc>, pid: u32) -> Result<()> { + let message = WireClientMessage::EntrypointStarted { pid }; + let mut writer = writer.lock().await; + write_json_line(&mut *writer, &message).await +} + +async fn write_json_line(writer: &mut W, value: &T) -> Result<()> +where + W: AsyncWrite + Unpin + Send, + T: Serialize + Sync, +{ + let bytes = serde_json::to_vec(value).into_diagnostic()?; + writer.write_all(&bytes).await.into_diagnostic()?; + writer.write_all(b"\n").await.into_diagnostic()?; + writer.flush().await.into_diagnostic()?; + Ok(()) +} + +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] +fn decode_client_message(line: &str) -> Result { + serde_json::from_str(line) + .into_diagnostic() + .wrap_err("failed to decode sidecar client message") +} + +fn decode_server_message(line: &str) -> Result { + serde_json::from_str(line) + .into_diagnostic() + .wrap_err("failed to decode sidecar server message") +} + +#[cfg(test)] +mod tests { + use super::*; + use openshell_core::proto::SandboxPolicy; + + fn current_peer() -> ExpectedPeer { + ExpectedPeer { + uid: nix::unistd::Uid::current().as_raw(), + gid: nix::unistd::Gid::current().as_raw(), + } + } + + #[tokio::test] + async fn bootstrap_round_trips_policy_and_provider_env() { + let dir = tempfile::tempdir().unwrap(); + let socket = dir.path().join("control.sock"); + let mut env = HashMap::new(); + env.insert("GITHUB_TOKEN".to_string(), "secret".to_string()); + let bootstrap = BootstrapData { + policy_proto: SandboxPolicy { + version: 7, + ..SandboxPolicy::default() + }, + provider_env_revision: 3, + provider_child_env: env.clone(), + proxy_ca_cert_path: Some(PathBuf::from("/tmp/ca.pem")), + proxy_ca_bundle_path: Some(PathBuf::from("/tmp/bundle.pem")), + }; + + let _server = spawn_server(&socket, bootstrap, current_peer()).unwrap(); + let (received, _connection) = connect_process_client(&socket, Duration::from_secs(1)) + .await + .unwrap(); + + assert_eq!(received.policy_proto.version, 7); + assert_eq!(received.provider_env_revision, 3); + assert_eq!(received.provider_child_env, env); + assert_eq!( + received.proxy_ca_cert_path, + Some(PathBuf::from("/tmp/ca.pem")) + ); + assert_eq!( + received.proxy_ca_bundle_path, + Some(PathBuf::from("/tmp/bundle.pem")) + ); + } + + #[tokio::test] + async fn entrypoint_started_is_delivered_to_server() { + let dir = tempfile::tempdir().unwrap(); + let socket = dir.path().join("control.sock"); + let server = spawn_server( + &socket, + BootstrapData { + policy_proto: SandboxPolicy::default(), + provider_env_revision: 0, + provider_child_env: HashMap::new(), + proxy_ca_cert_path: None, + proxy_ca_bundle_path: None, + }, + current_peer(), + ) + .unwrap(); + let mut entrypoint_rx = server.into_entrypoint_receiver(); + let (_bootstrap, connection) = connect_process_client(&socket, Duration::from_secs(1)) + .await + .unwrap(); + + let anchor = tokio::time::timeout(Duration::from_secs(1), entrypoint_rx.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(anchor.pid, std::process::id()); + assert!(!anchor.start_session); + + send_entrypoint_started(&connection.writer, 4242) + .await + .unwrap(); + + let started = tokio::time::timeout(Duration::from_secs(1), entrypoint_rx.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(started.pid, 4242); + assert!(started.start_session); + } + + #[tokio::test] + async fn second_control_client_is_rejected_after_authoritative_bootstrap() { + let dir = tempfile::tempdir().unwrap(); + let socket = dir.path().join("control.sock"); + let _server = spawn_server( + &socket, + BootstrapData { + policy_proto: SandboxPolicy::default(), + provider_env_revision: 0, + provider_child_env: HashMap::new(), + proxy_ca_cert_path: None, + proxy_ca_bundle_path: None, + }, + current_peer(), + ) + .unwrap(); + + let (_bootstrap, _connection) = connect_process_client(&socket, Duration::from_secs(1)) + .await + .unwrap(); + + let err = tokio::net::UnixStream::connect(&socket) + .await + .expect_err("control listener must be removed after the first bootstrap"); + assert!( + matches!( + err.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::ConnectionRefused + ), + "unexpected second-client error: {err}" + ); + } + + #[tokio::test] + async fn authoritative_connection_task_ends_when_process_supervisor_disconnects() { + let dir = tempfile::tempdir().unwrap(); + let socket = dir.path().join("control.sock"); + let server = spawn_server( + &socket, + BootstrapData { + policy_proto: SandboxPolicy::default(), + provider_env_revision: 0, + provider_child_env: HashMap::new(), + proxy_ca_cert_path: None, + proxy_ca_bundle_path: None, + }, + current_peer(), + ) + .unwrap(); + let (_entrypoint_rx, connection_task) = server.into_runtime_parts(); + let (_bootstrap, connection) = connect_process_client(&socket, Duration::from_secs(1)) + .await + .unwrap(); + + drop(connection); + tokio::time::timeout(Duration::from_secs(1), connection_task) + .await + .expect("server must observe authoritative client disconnect") + .expect("control task must not panic"); + } + + #[tokio::test] + async fn process_client_reports_network_sidecar_restart() { + let dir = tempfile::tempdir().unwrap(); + let socket = dir.path().join("control.sock"); + let server = spawn_server( + &socket, + BootstrapData { + policy_proto: SandboxPolicy::default(), + provider_env_revision: 0, + provider_child_env: HashMap::new(), + proxy_ca_cert_path: None, + proxy_ca_bundle_path: None, + }, + current_peer(), + ) + .unwrap(); + let (_entrypoint_rx, connection_task) = server.into_runtime_parts(); + let (_bootstrap, connection) = connect_process_client(&socket, Duration::from_secs(1)) + .await + .unwrap(); + + connection_task.abort(); + let _ = connection_task.await; + tokio::time::timeout(Duration::from_secs(1), connection.closed) + .await + .expect("process supervisor must observe network sidecar disconnect") + .expect("disconnect notifier must remain live"); + } + + #[tokio::test] + async fn bootstrap_rejects_claimed_pid_that_does_not_match_peer_credentials() { + let dir = tempfile::tempdir().unwrap(); + let socket = dir.path().join("control.sock"); + let server = spawn_server( + &socket, + BootstrapData { + policy_proto: SandboxPolicy::default(), + provider_env_revision: 0, + provider_child_env: HashMap::new(), + proxy_ca_cert_path: None, + proxy_ca_bundle_path: None, + }, + current_peer(), + ) + .unwrap(); + let mut entrypoint_rx = server.into_entrypoint_receiver(); + + let mut stream = tokio::net::UnixStream::connect(&socket).await.unwrap(); + write_json_line( + &mut stream, + &WireClientMessage::BootstrapRequest { + supervisor_pid: std::process::id().saturating_add(1), + }, + ) + .await + .unwrap(); + + assert!( + tokio::time::timeout(Duration::from_secs(1), entrypoint_rx.recv()) + .await + .unwrap() + .is_none(), + "mismatched bootstrap must not publish a process anchor" + ); + } + + #[test] + fn malformed_client_message_is_rejected() { + let err = decode_client_message("not-json").unwrap_err(); + assert!( + err.to_string() + .contains("failed to decode sidecar client message") + ); + } +} diff --git a/crates/openshell-supervisor-network/data/sandbox-policy.rego b/crates/openshell-supervisor-network/data/sandbox-policy.rego index efcdf07320..d70c69b748 100644 --- a/crates/openshell-supervisor-network/data/sandbox-policy.rego +++ b/crates/openshell-supervisor-network/data/sandbox-policy.rego @@ -19,6 +19,10 @@ allow_network if { network_policy_for_request } +binary_identity_required if { + object.get(object.get(data, "runtime", {}), "require_binary_identity", true) +} + # --- Deny reasons (specific diagnostics for debugging policy denials) --- deny_reason := "missing input.network" if { @@ -131,6 +135,12 @@ endpoint_allowed(policy, network) if { endpoint.ports[_] == network.port } +# Binary matching can be relaxed by trusted runtime configuration. In that +# mode, network policies are endpoint/L7 scoped and ignore policy.binaries. +binary_allowed(_, _) if { + not binary_identity_required +} + # Binary matching: exact path. # SHA256 integrity is enforced in Rust via trust-on-first-use (TOFU) cache, # not in Rego. The proxy computes and caches binary hashes at runtime. @@ -161,6 +171,10 @@ binary_allowed(policy, exec) if { glob.match(b.path, ["/"], p) } +user_declared_binary_allowed(_, _) if { + not binary_identity_required +} + user_declared_binary_allowed(policy, exec) if { some b b := policy.binaries[_] diff --git a/crates/openshell-supervisor-network/src/identity.rs b/crates/openshell-supervisor-network/src/identity.rs index fce568f41e..5e89c35031 100644 --- a/crates/openshell-supervisor-network/src/identity.rs +++ b/crates/openshell-supervisor-network/src/identity.rs @@ -100,23 +100,34 @@ impl BinaryIdentityCache { /// Returns `Ok(hash)` if it matches, `Err` if the hash changed (binary tampered). #[cfg_attr(not(target_os = "linux"), allow(dead_code))] pub fn verify_or_cache(&self, path: &Path) -> Result { - self.verify_or_cache_with_hasher(path, procfs::file_sha256) + self.verify_or_cache_with_paths(path, path, procfs::file_sha256) } - fn verify_or_cache_with_hasher(&self, path: &Path, mut hash_file: F) -> Result + #[cfg(target_os = "linux")] + pub fn verify_or_cache_process_exe(&self, display_path: &Path, pid: u32) -> Result { + let proc_exe = PathBuf::from(format!("/proc/{pid}/exe")); + self.verify_or_cache_with_paths(display_path, &proc_exe, procfs::file_sha256) + } + + fn verify_or_cache_with_paths( + &self, + cache_path: &Path, + access_path: &Path, + mut hash_file: F, + ) -> Result where F: FnMut(&Path) -> Result, { let start = std::time::Instant::now(); - let metadata = std::fs::metadata(path) - .map_err(|error| miette::miette!("Failed to stat {}: {error}", path.display()))?; + let metadata = std::fs::metadata(access_path) + .map_err(|error| miette::miette!("Failed to stat {}: {error}", cache_path.display()))?; let fingerprint = FileFingerprint::from_metadata(&metadata); let cached = self .hashes .lock() .map_err(|_| miette::miette!("Binary identity cache lock poisoned"))? - .get(path) + .get(cache_path) .cloned(); if let Some(cached_binary) = &cached @@ -125,7 +136,7 @@ impl BinaryIdentityCache { debug!( " verify_or_cache: {}ms CACHE HIT path={}", start.elapsed().as_millis(), - path.display() + cache_path.display() ); return Ok(cached_binary.hash.clone()); } @@ -133,29 +144,29 @@ impl BinaryIdentityCache { debug!( " verify_or_cache: CACHE MISS size={} path={}", metadata.len(), - path.display() + cache_path.display() ); - let current_hash = hash_file(path)?; + let current_hash = hash_file(access_path)?; let mut hashes = self .hashes .lock() .map_err(|_| miette::miette!("Binary identity cache lock poisoned"))?; - if let Some(existing) = hashes.get(path) + if let Some(existing) = hashes.get(cache_path) && existing.hash != current_hash { return Err(miette::miette!( "Binary integrity violation: {} hash changed (cached: {}, current: {})", - path.display(), + cache_path.display(), existing.hash, current_hash )); } hashes.insert( - path.to_path_buf(), + cache_path.to_path_buf(), CachedBinary { hash: current_hash.clone(), fingerprint, @@ -165,7 +176,7 @@ impl BinaryIdentityCache { debug!( " verify_or_cache TOTAL (cold): {}ms path={}", start.elapsed().as_millis(), - path.display() + cache_path.display() ); Ok(current_hash) @@ -212,13 +223,13 @@ mod tests { let mut hash_calls = 0; let hash1 = cache - .verify_or_cache_with_hasher(tmp.path(), |path| { + .verify_or_cache_with_paths(tmp.path(), tmp.path(), |path| { hash_calls += 1; procfs::file_sha256(path) }) .unwrap(); let hash2 = cache - .verify_or_cache_with_hasher(tmp.path(), |path| { + .verify_or_cache_with_paths(tmp.path(), tmp.path(), |path| { hash_calls += 1; procfs::file_sha256(path) }) @@ -238,7 +249,7 @@ mod tests { let mut hash_calls = 0; let hash1 = cache - .verify_or_cache_with_hasher(tmp.path(), |path| { + .verify_or_cache_with_paths(tmp.path(), tmp.path(), |path| { hash_calls += 1; procfs::file_sha256(path) }) @@ -254,7 +265,7 @@ mod tests { .unwrap(); let hash2 = cache - .verify_or_cache_with_hasher(tmp.path(), |path| { + .verify_or_cache_with_paths(tmp.path(), tmp.path(), |path| { hash_calls += 1; procfs::file_sha256(path) }) @@ -275,7 +286,7 @@ mod tests { let mut hash_calls = 0; cache - .verify_or_cache_with_hasher(&path, |path| { + .verify_or_cache_with_paths(&path, &path, |path| { hash_calls += 1; procfs::file_sha256(path) }) @@ -292,7 +303,7 @@ mod tests { .set_modified(original_mtime) .unwrap(); - let result = cache.verify_or_cache_with_hasher(&path, |path| { + let result = cache.verify_or_cache_with_paths(&path, &path, |path| { hash_calls += 1; procfs::file_sha256(path) }); @@ -301,6 +312,28 @@ mod tests { assert_eq!(hash_calls, 2); } + #[test] + fn display_path_can_differ_from_access_path() { + let mut tmp = tempfile::NamedTempFile::new().unwrap(); + tmp.write_all(b"binary content").unwrap(); + tmp.flush().unwrap(); + let display_path = Path::new("/usr/bin/python3"); + + let cache = BinaryIdentityCache::new(); + let hash = cache + .verify_or_cache_with_paths(display_path, tmp.path(), procfs::file_sha256) + .unwrap(); + + assert!(!hash.is_empty()); + assert!( + cache + .hashes + .lock() + .unwrap() + .contains_key(Path::new("/usr/bin/python3")) + ); + } + #[test] fn hash_mismatch_returns_error() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index aaa79f9281..829c63caa4 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -18,6 +18,7 @@ use std::sync::{ Arc, Mutex, atomic::{AtomicU64, Ordering}, }; +use tracing::info; /// Baked-in rego rules for OPA policy evaluation. /// These rules define the network access decision logic and static config @@ -55,6 +56,49 @@ pub struct NetworkInput { pub cmdline_paths: Vec, } +pub(crate) fn network_binary_identity_required() -> bool { + std::env::var(openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY).map_or(true, |value| { + !matches!( + value.as_str(), + "relaxed" | "disabled" | "endpoint-only" | "false" | "0" + ) + }) +} + +fn inject_runtime_policy_data(data: &mut serde_json::Value, require_binary_identity: bool) { + let Some(obj) = data.as_object_mut() else { + return; + }; + obj.insert( + "runtime".to_string(), + serde_json::json!({ + "require_binary_identity": require_binary_identity, + }), + ); +} + +fn emit_binary_identity_mode(require_binary_identity: bool, source: &str) { + info!( + require_binary_identity, + source, "Configured OPA runtime binary identity mode" + ); + openshell_ocsf::ocsf_emit!( + openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(openshell_ocsf::SeverityId::Informational) + .status(openshell_ocsf::StatusId::Success) + .state(openshell_ocsf::StateId::Enabled, "configured") + .unmapped( + "require_binary_identity", + serde_json::json!(require_binary_identity) + ) + .unmapped("source", serde_json::json!(source)) + .message(format!( + "OPA runtime binary identity mode configured [source:{source} require_binary_identity:{require_binary_identity}]" + )) + .build() + ); +} + /// Sandbox configuration extracted from OPA data at startup. pub struct SandboxConfig { pub filesystem: FilesystemPolicy, @@ -146,7 +190,9 @@ impl OpaEngine { engine .add_policy_from_file(policy_path) .map_err(|e| miette::miette!("{e}"))?; - let data_json = preprocess_yaml_data(&yaml_str)?; + let require_binary_identity = network_binary_identity_required(); + emit_binary_identity_mode(require_binary_identity, "files"); + let data_json = preprocess_yaml_data(&yaml_str, require_binary_identity)?; engine .add_data_json(&data_json) .map_err(|e| miette::miette!("{e}"))?; @@ -160,11 +206,24 @@ impl OpaEngine { /// /// Preprocesses the YAML data to expand access presets and validate L7 config. pub fn from_strings(policy: &str, data_yaml: &str) -> Result { + Self::from_strings_with_binary_identity_required( + policy, + data_yaml, + network_binary_identity_required(), + ) + } + + pub(crate) fn from_strings_with_binary_identity_required( + policy: &str, + data_yaml: &str, + require_binary_identity: bool, + ) -> Result { let mut engine = regorus::Engine::new(); engine .add_policy("policy.rego".into(), policy.into()) .map_err(|e| miette::miette!("{e}"))?; - let data_json = preprocess_yaml_data(data_yaml)?; + emit_binary_identity_mode(require_binary_identity, "strings"); + let data_json = preprocess_yaml_data(data_yaml, require_binary_identity)?; engine .add_data_json(&data_json) .map_err(|e| miette::miette!("{e}"))?; @@ -193,11 +252,25 @@ impl OpaEngine { /// gap between user-specified symlink paths (e.g., `/usr/bin/python3`) and /// kernel-resolved canonical paths (e.g., `/usr/bin/python3.11`). pub fn from_proto_with_pid(proto: &ProtoSandboxPolicy, entrypoint_pid: u32) -> Result { + Self::from_proto_with_pid_and_binary_identity_required( + proto, + entrypoint_pid, + network_binary_identity_required(), + ) + } + + fn from_proto_with_pid_and_binary_identity_required( + proto: &ProtoSandboxPolicy, + entrypoint_pid: u32, + require_binary_identity: bool, + ) -> Result { + emit_binary_identity_mode(require_binary_identity, "proto"); let data_json_str = proto_to_opa_data_json(proto, entrypoint_pid); // Parse back to Value for preprocessing, then re-serialize let mut data: serde_json::Value = serde_json::from_str(&data_json_str) .map_err(|e| miette::miette!("internal: failed to parse proto JSON: {e}"))?; + inject_runtime_policy_data(&mut data, require_binary_identity); // Validate BEFORE expanding presets let (errors, warnings) = crate::l7::validate_l7_policies(&data); @@ -720,9 +793,10 @@ fn parse_process_policy(val: ®orus::Value) -> ProcessPolicy { } /// Preprocess YAML policy data: parse, normalize, validate, expand access presets, return JSON. -fn preprocess_yaml_data(yaml_str: &str) -> Result { +fn preprocess_yaml_data(yaml_str: &str, require_binary_identity: bool) -> Result { let mut data: serde_json::Value = serde_yml::from_str(yaml_str) .map_err(|e| miette::miette!("failed to parse YAML data: {e}"))?; + inject_runtime_policy_data(&mut data, require_binary_identity); // Normalize port → ports for all endpoints so Rego always sees "ports" array. normalize_endpoint_ports(&mut data); @@ -2298,6 +2372,88 @@ process: assert!(eval_l7(&engine, &input)); } + #[test] + fn l7_get_allowed_by_rules_when_binary_identity_relaxed() { + let engine = + OpaEngine::from_strings_with_binary_identity_required(TEST_POLICY, L7_TEST_DATA, false) + .expect("Failed to load relaxed L7 test data"); + let mut input = l7_input("api.example.com", 8080, "GET", "/repos/myorg/foo"); + input["exec"]["path"] = "".into(); + assert!(eval_l7(&engine, &input)); + } + + #[test] + fn relaxed_binary_identity_preserves_matched_policy_and_l7_for_proto() { + let mut network_policies = std::collections::HashMap::new(); + network_policies.insert( + "test_l7".to_string(), + NetworkPolicyRule { + name: "test_l7".to_string(), + endpoints: vec![NetworkEndpoint { + host: "host.k3d.internal".to_string(), + port: 56123, + protocol: "rest".to_string(), + enforcement: "enforce".to_string(), + rules: vec![L7Rule { + allow: Some(L7Allow { + method: "GET".to_string(), + path: "/allowed".to_string(), + command: String::new(), + query: std::collections::HashMap::new(), + operation_type: String::new(), + operation_name: String::new(), + fields: Vec::new(), + params: std::collections::HashMap::new(), + }), + }], + allowed_ips: vec!["192.168.0.0/16".to_string()], + ..Default::default() + }], + binaries: vec![NetworkBinary { + path: "/usr/bin/curl".to_string(), + ..Default::default() + }], + }, + ); + let proto = ProtoSandboxPolicy { + version: 1, + filesystem: Some(ProtoFs { + include_workdir: true, + read_only: vec![], + read_write: vec![], + }), + landlock: Some(openshell_core::proto::LandlockPolicy { + compatibility: "best_effort".to_string(), + }), + process: Some(ProtoProc { + run_as_user: "sandbox".to_string(), + run_as_group: "sandbox".to_string(), + }), + network_policies, + }; + let engine = OpaEngine::from_proto_with_pid_and_binary_identity_required(&proto, 0, false) + .expect("engine from relaxed proto"); + let network_input = NetworkInput { + host: "host.k3d.internal".into(), + port: 56123, + binary_path: PathBuf::new(), + binary_sha256: String::new(), + ancestors: vec![], + cmdline_paths: vec![], + }; + let action = engine.evaluate_network_action(&network_input).unwrap(); + assert_eq!( + action, + NetworkAction::Allow { + matched_policy: Some("test_l7".to_string()) + } + ); + + let mut input = l7_input("host.k3d.internal", 56123, "GET", "/allowed"); + input["exec"]["path"] = "".into(); + assert!(eval_l7(&engine, &input)); + } + #[test] fn l7_post_allowed_by_rules() { let engine = l7_engine(); @@ -4626,6 +4782,46 @@ process: ); } + #[test] + fn relaxed_binary_identity_allows_declared_endpoint_without_binary_match() { + let engine = OpaEngine::from_strings_with_binary_identity_required( + TEST_POLICY, + INFERENCE_TEST_DATA, + false, + ) + .expect("Failed to load relaxed binary identity test data"); + let input = NetworkInput { + host: "api.anthropic.com".into(), + port: 443, + binary_path: PathBuf::from("/tmp/unlisted-agent"), + binary_sha256: "unused".into(), + ancestors: vec![], + cmdline_paths: vec![], + }; + + let action = engine.evaluate_network_action(&input).unwrap(); + assert_eq!( + action, + NetworkAction::Allow { + matched_policy: Some("claude_code".to_string()) + }, + ); + assert!( + engine.query_exact_declared_endpoint_host(&input).unwrap(), + "relaxed identity should preserve exact declared endpoint handling" + ); + + let undeclared = NetworkInput { + host: "api.openai.com".into(), + ..input + }; + let action = engine.evaluate_network_action(&undeclared).unwrap(); + assert!( + matches!(action, NetworkAction::Deny { .. }), + "relaxed identity must not allow undeclared endpoints" + ); + } + #[test] fn unknown_endpoint_returns_deny() { let engine = inference_engine(); diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 6d60536362..c0c33e6fd1 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -42,6 +42,8 @@ const TUNNEL_PROTOCOL_PEEK_POLL: std::time::Duration = std::time::Duration::from const TUNNEL_PROTOCOL_PEEK_POLL: std::time::Duration = std::time::Duration::from_millis(1); const INFERENCE_LOCAL_HOST: &str = "inference.local"; const INFERENCE_LOCAL_PORT: u16 = 443; +#[cfg(target_os = "linux")] +const SIDECAR_SUPERVISOR_TOPOLOGY: &str = "sidecar"; /// Hostnames injected by compute drivers as `/etc/hosts` aliases for the host /// machine. Traffic to these names is eligible for the trusted-gateway SSRF @@ -1426,7 +1428,7 @@ fn resolve_owner_identity( })?; let bin_hash = identity_cache - .verify_or_cache(&bin_path) + .verify_or_cache_process_exe(&bin_path, owner_pid) .map_err(|e| IdentityError { reason: format!("binary integrity check failed: {e}"), binary: Some(bin_path.clone()), @@ -1434,11 +1436,15 @@ fn resolve_owner_identity( ancestors: vec![], })?; - let ancestors = crate::procfs::collect_ancestor_binaries(owner_pid, entrypoint_pid); + let ancestor_identities = collect_ancestor_identities(owner_pid, entrypoint_pid); + let ancestors: Vec = ancestor_identities + .iter() + .map(|(_, path)| path.clone()) + .collect(); - for ancestor in &ancestors { + for (ancestor_pid, ancestor) in &ancestor_identities { identity_cache - .verify_or_cache(ancestor) + .verify_or_cache_process_exe(ancestor, *ancestor_pid) .map_err(|e| IdentityError { reason: format!( "ancestor integrity check failed for {}: {e}", @@ -1463,6 +1469,31 @@ fn resolve_owner_identity( }) } +#[cfg(target_os = "linux")] +fn collect_ancestor_identities(start_pid: u32, stop_pid: u32) -> Vec<(u32, PathBuf)> { + const MAX_DEPTH: usize = 64; + let mut ancestors = Vec::new(); + let mut current = start_pid; + + for _ in 0..MAX_DEPTH { + let parent_pid = match crate::procfs::read_ppid(current) { + Some(parent) if parent > 0 && parent != current => parent, + _ => break, + }; + + if let Ok(path) = crate::procfs::binary_path(parent_pid.cast_signed()) { + ancestors.push((parent_pid, path)); + } + + if parent_pid == stop_pid || parent_pid == 1 { + break; + } + current = parent_pid; + } + + ancestors +} + /// Resolve the identity of the process owning a TCP peer connection. /// /// Walks `/proc//net/tcp` to find the socket inode, locates @@ -1472,10 +1503,10 @@ fn resolve_owner_identity( /// /// This is the identity-resolution block of [`evaluate_opa_tcp`] extracted /// into a standalone helper so it can be exercised by Linux-only regression -/// tests without a full OPA engine. The key invariant under test is that on -/// a hot-swap of the peer binary, the failure mode is -/// `"Binary integrity violation"` (from the identity cache) rather than -/// `"Failed to stat ... (deleted)"` (from the kernel-tainted path). +/// tests without a full OPA engine. The key hot-swap invariant under test is +/// that display paths are stripped for policy/logging, while integrity hashing +/// reads the live executable via `/proc//exe` instead of the replacement +/// file that now exists at the display path. #[cfg(target_os = "linux")] fn resolve_process_identity( entrypoint_pid: u32, @@ -1573,8 +1604,17 @@ fn evaluate_opa_tcp( } }; - let pid = entrypoint_pid.load(Ordering::Acquire); - if pid == 0 { + if !crate::opa::network_binary_identity_required() { + let result = evaluate_endpoint_only_opa(engine, host, port); + debug!( + "evaluate_opa_tcp endpoint-only: host={host} port={port} action={:?}", + result.action + ); + return result; + } + + let entrypoint_pid = entrypoint_pid.load(Ordering::Acquire); + let Some(proc_net_anchor_pid) = proc_net_anchor_pid(entrypoint_pid) else { return deny( "entrypoint process not yet spawned".into(), None, @@ -1582,12 +1622,12 @@ fn evaluate_opa_tcp( vec![], vec![], ); - } + }; let total_start = std::time::Instant::now(); let peer_port = peer_addr.port(); - let identity = match resolve_process_identity(pid, peer_port, identity_cache) { + let identity = match resolve_process_identity(proc_net_anchor_pid, peer_port, identity_cache) { Ok(id) => id, Err(err) => { return deny( @@ -1641,6 +1681,52 @@ fn evaluate_opa_tcp( result } +#[cfg(target_os = "linux")] +fn proc_net_anchor_pid(entrypoint_pid: u32) -> Option { + if entrypoint_pid != 0 { + return Some(entrypoint_pid); + } + sidecar_topology_enabled().then(std::process::id) +} + +#[cfg(target_os = "linux")] +fn sidecar_topology_enabled() -> bool { + std::env::var(openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY) + .is_ok_and(|value| value == SIDECAR_SUPERVISOR_TOPOLOGY) +} + +fn evaluate_endpoint_only_opa(engine: &OpaEngine, host: &str, port: u16) -> ConnectDecision { + let input = crate::opa::NetworkInput { + host: host.to_string(), + port, + binary_path: PathBuf::new(), + binary_sha256: String::new(), + ancestors: vec![], + cmdline_paths: vec![], + }; + + match engine.evaluate_network_action_with_generation(&input) { + Ok((action, generation)) => ConnectDecision { + action, + generation, + binary: None, + binary_pid: None, + ancestors: vec![], + cmdline_paths: vec![], + }, + Err(e) => ConnectDecision { + action: NetworkAction::Deny { + reason: format!("policy evaluation error: {e}"), + }, + generation: engine.current_generation(), + binary: None, + binary_pid: None, + ancestors: vec![], + cmdline_paths: vec![], + }, + } +} + /// Non-Linux stub: OPA identity binding requires /proc. #[cfg(not(target_os = "linux"))] fn evaluate_opa_tcp( @@ -1648,9 +1734,13 @@ fn evaluate_opa_tcp( engine: &OpaEngine, _identity_cache: &BinaryIdentityCache, _entrypoint_pid: &AtomicU32, - _host: &str, - _port: u16, + host: &str, + port: u16, ) -> ConnectDecision { + if !crate::opa::network_binary_identity_required() { + return evaluate_endpoint_only_opa(engine, host, port); + } + ConnectDecision { action: NetworkAction::Deny { reason: "identity binding unavailable on this platform".into(), @@ -2152,14 +2242,24 @@ fn query_l7_route_snapshot( }; match engine.query_endpoint_configs_with_generation(&input) { - Ok((vals, generation)) => Some(L7RouteSnapshot { - configs: vals + Ok((vals, generation)) => { + let configs: Vec<_> = vals .into_iter() .filter_map(|val| crate::l7::parse_l7_config(&val)) .map(|config| L7ConfigSnapshot { config }) - .collect(), - generation, - }), + .collect(); + debug!( + host, + port, + generation, + config_count = configs.len(), + "Forward proxy L7 route lookup complete" + ); + Some(L7RouteSnapshot { + configs, + generation, + }) + } Err(e) => { let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Fail) @@ -3337,10 +3437,29 @@ async fn handle_forward_proxy( } }; let policy_str = matched_policy.as_deref().unwrap_or("-"); + debug!( + host = %host_lc, + port, + binary = %binary_str, + binary_pid = %pid_str, + matched_policy = %policy_str, + decision_generation = decision.generation, + current_generation = opa_engine.current_generation(), + action = ?decision.action, + "Forward proxy L4 policy decision" + ); let sandbox_entrypoint_pid = entrypoint_pid.load(Ordering::Acquire); let forward_generation_guard = match opa_engine.generation_guard(decision.generation) { Ok(guard) => guard, Err(e) => { + warn!( + host = %host_lc, + port, + decision_generation = decision.generation, + current_generation = opa_engine.current_generation(), + error = %e, + "Forward proxy rejected request because policy generation changed after L4 decision" + ); emit_l7_tunnel_close_after_policy_change(&host_lc, port, e); emit_activity_simple(activity_tx, true, "policy_stale"); respond( @@ -3401,6 +3520,15 @@ async fn handle_forward_proxy( && !route.configs.is_empty() { if route.generation != forward_generation_guard.captured_generation() { + warn!( + host = %host_lc, + port, + decision_generation = decision.generation, + guard_generation = forward_generation_guard.captured_generation(), + route_generation = route.generation, + current_generation = opa_engine.current_generation(), + "Forward proxy rejected request because L7 route lookup used a different policy generation" + ); emit_l7_tunnel_close_after_policy_change( &host_lc, port, @@ -3426,6 +3554,14 @@ async fn handle_forward_proxy( let tunnel_engine = match opa_engine.clone_engine_for_tunnel(route.generation) { Ok(engine) => engine, Err(e) => { + warn!( + host = %host_lc, + port, + route_generation = route.generation, + current_generation = opa_engine.current_generation(), + error = %e, + "Forward proxy rejected request because L7 tunnel engine could not be cloned" + ); emit_l7_tunnel_close_after_policy_change(&host_lc, port, e); emit_activity_simple(activity_tx, true, "policy_stale"); respond( @@ -4105,6 +4241,14 @@ async fn handle_forward_proxy( }; if let Err(e) = forward_generation_guard.ensure_current() { + warn!( + host = %host_lc, + port, + captured_generation = forward_generation_guard.captured_generation(), + current_generation = forward_generation_guard.current_generation(), + error = %e, + "Forward proxy rejected request because policy changed before upstream connect" + ); emit_l7_tunnel_close_after_policy_change(&host_lc, port, e); emit_activity_simple(activity_tx, true, "policy_stale"); respond( @@ -4243,6 +4387,14 @@ async fn handle_forward_proxy( }; if let Err(e) = forward_generation_guard.ensure_current() { + warn!( + host = %host_lc, + port, + captured_generation = forward_generation_guard.captured_generation(), + current_generation = forward_generation_guard.current_generation(), + error = %e, + "Forward proxy rejected request because policy changed before relay" + ); emit_l7_tunnel_close_after_policy_change(&host_lc, port, e); respond( client, @@ -4379,6 +4531,46 @@ mod tests { use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; + #[test] + fn endpoint_only_opa_allows_declared_endpoint_without_process_identity() { + let policy = include_str!("../data/sandbox-policy.rego"); + let data = r#" +version: 1 +network_policies: + test_l7: + name: test_l7 + endpoints: + - host: host.k3d.internal + port: 56123 + protocol: rest + enforcement: enforce + rules: + - allow: + method: GET + path: /allowed + binaries: + - path: /usr/bin/curl +"#; + let engine = OpaEngine::from_strings_with_binary_identity_required(policy, data, false) + .expect("relaxed engine"); + + let decision = evaluate_endpoint_only_opa(&engine, "host.k3d.internal", 56123); + assert_eq!( + decision.action, + NetworkAction::Allow { + matched_policy: Some("test_l7".to_string()), + } + ); + assert!(decision.binary.is_none()); + assert!(decision.ancestors.is_empty()); + + let denied = evaluate_endpoint_only_opa(&engine, "api.example.com", 443); + assert!( + matches!(denied.action, NetworkAction::Deny { .. }), + "endpoint-only mode must still deny undeclared endpoints" + ); + } + fn websocket_l7_config( protocol: crate::l7::L7Protocol, websocket_credential_rewrite: bool, @@ -7992,27 +8184,23 @@ network_policies: assert_eq!(resp_str[body_start..].len(), cl); } - /// End-to-end regression for the `docker cp` hot-swap hazard that - /// motivated `binary_path()` stripping the kernel's `" (deleted)"` - /// suffix (PR #844). + /// End-to-end regression for the `docker cp` hot-swap hazard around + /// unlinked process executables. /// - /// Before the strip, the identity-resolution chain inside - /// `evaluate_opa_tcp` failed with `"Failed to stat - /// /opt/openshell/bin/openshell-sandbox (deleted)"` because - /// `BinaryIdentityCache::verify_or_cache()` tried to `metadata()` the - /// tainted path. That masked the real security signal: a live process - /// was now bound to a *different* binary on disk than the one that was - /// TOFU-cached. After the strip, `binary_path()` returns a path that - /// stats fine, the cache rehashes the new bytes, and the hash mismatch - /// surfaces as a `Binary integrity violation` error — the contract this - /// PR is trying to establish. + /// `binary_path()` strips the kernel's `" (deleted)"` suffix so policy + /// identity and logs use a clean display path. Integrity verification must + /// not hash that display path after a hot-swap, because it may now point to + /// unrelated replacement bytes. It hashes `/proc//exe` instead, which + /// resolves to the live executable inode even after the original path was + /// unlinked. /// /// Test shape (from the review comment on the initial PR): /// 1. Start a `TcpListener` in the test process. /// 2. Copy `/bin/bash` to a temp path we control. /// 3. Prime `BinaryIdentityCache` with that temp binary's hash. /// 4. Spawn the temp bash as a child with a `/dev/tcp` one-liner that - /// opens a real TCP connection to the listener and holds it open. + /// opens a real TCP connection to the listener and holds it open + /// inside the bash process. /// 5. Accept the connection on the listener side and capture the peer's /// ephemeral port — that's what `resolve_process_identity` uses to /// walk `/proc/net/tcp` back to the child PID. @@ -8022,13 +8210,12 @@ network_policies: /// now readlink to `" (deleted)"` OR the overwritten file, depending /// on whether the filesystem reused the inode. /// 7. Call `resolve_process_identity` and assert: - /// - the error reason contains `"Binary integrity violation"` (the - /// cache detected the tampered on-disk bytes), and - /// - the error reason does NOT contain `"Failed to stat"` or - /// `"(deleted)"` (the old pre-strip failure mode). + /// - identity resolution succeeds using the live executable hash, and + /// - the returned display path does not contain the kernel-added + /// `"(deleted)"` suffix. #[cfg(target_os = "linux")] #[test] - fn resolve_process_identity_surfaces_binary_integrity_violation_on_hot_swap() { + fn resolve_process_identity_hashes_live_exe_after_hot_swap() { use crate::identity::BinaryIdentityCache; use std::io::Read; use std::net::TcpListener; @@ -8060,9 +8247,12 @@ network_policies: assert!(!v1_hash.is_empty()); // 4. Spawn the temp bash with a /dev/tcp one-liner that opens a real - // connection to the listener and sleeps to keep it open. The - // `read -t` blocks on stdin so the shell stays resident. - let script = format!("exec 3<>/dev/tcp/127.0.0.1/{listener_port}; sleep 30 <&3"); + // connection to the listener and blocks in bash's `read` builtin + // to keep it open. Do not use an external command like `sleep`: + // it inherits the socket fd and intentionally trips the shared + // socket ambiguity guard instead of exercising the hot-swap path. + let script = + format!("exec 3<>/dev/tcp/127.0.0.1/{listener_port}; read -r -t 30 _ <&3 || true"); let mut child = Command::new(&bash_v1) .arg("-c") .arg(&script) @@ -8106,10 +8296,11 @@ network_policies: std::fs::write(&bash_v1, tampered_bytes).expect("write replacement bytes"); // 7. Resolve identity through the real helper and assert the - // contract: we want "Binary integrity violation", not - // "Failed to stat ... (deleted)". + // contract: hash the live executable via /proc//exe while + // returning a clean display path for policy/logging. let test_pid = std::process::id(); let result = resolve_process_identity(test_pid, peer_port, &cache); + let child_pid = child.id(); // Always clean up the child before asserting so a failure doesn't // leak a sleeping process across test runs. @@ -8117,40 +8308,29 @@ network_policies: let _ = child.wait(); match result { - Ok(_) => panic!( - "resolve_process_identity unexpectedly succeeded after hot-swap; \ - the cache should have detected the tampered on-disk bytes" - ), - Err(err) => { - assert!( - err.reason.contains("Binary integrity violation"), - "expected 'Binary integrity violation' error, got: {}", - err.reason + Ok(identity) => { + assert_eq!( + identity.binary_pid, child_pid, + "expected the hot-swapped bash child to own the socket" ); - assert!( - !err.reason.contains("Failed to stat"), - "pre-PR-#844 failure mode leaked: {}", - err.reason + assert_eq!( + identity.bin_path, bash_v1, + "expected stripped display path to remain the original binary path" ); assert!( - !err.reason.contains("(deleted)"), - "resolved path still contains '(deleted)' suffix: {}", - err.reason + !identity.bin_path.to_string_lossy().contains("(deleted)"), + "resolved binary path still tainted: {}", + identity.bin_path.display() ); - // The binary field should be populated — we did resolve a - // path before failing. - assert!( - err.binary.is_some(), - "expected resolved binary path on integrity failure" + assert_eq!( + identity.bin_hash, v1_hash, + "expected integrity hash from the live executable, not replacement bytes" ); - if let Some(path) = &err.binary { - assert!( - !path.to_string_lossy().contains("(deleted)"), - "resolved binary path still tainted: {}", - path.display() - ); - } } + Err(err) => panic!( + "resolve_process_identity failed after hot-swap; expected live-exe identity: {}", + err.reason + ), } } diff --git a/crates/openshell-supervisor-network/src/run.rs b/crates/openshell-supervisor-network/src/run.rs index 9553e06736..8e17758bd6 100644 --- a/crates/openshell-supervisor-network/src/run.rs +++ b/crates/openshell-supervisor-network/src/run.rs @@ -201,7 +201,9 @@ pub async fn run_networking( let (tls_state, ca_file_paths) = if matches!(policy.network.mode, NetworkMode::Proxy) { match SandboxCa::generate() { Ok(ca) => { - let tls_dir = std::path::Path::new("/etc/openshell-tls"); + let tls_dir = std::env::var(openshell_core::sandbox_env::PROXY_TLS_DIR) + .unwrap_or_else(|_| "/etc/openshell-tls".to_string()); + let tls_dir = std::path::Path::new(&tls_dir); let system_ca_bundle = read_system_ca_bundle(); match write_ca_files(&ca, tls_dir, &system_ca_bundle) { Ok(paths) => { diff --git a/crates/openshell-supervisor-process/src/lib.rs b/crates/openshell-supervisor-process/src/lib.rs index 1a3efd733d..842b62f9df 100644 --- a/crates/openshell-supervisor-process/src/lib.rs +++ b/crates/openshell-supervisor-process/src/lib.rs @@ -19,6 +19,8 @@ pub mod skills; pub mod ssh; pub mod supervisor_session; +mod unix_socket; + #[cfg(target_os = "linux")] pub mod bypass_monitor; #[cfg(target_os = "linux")] diff --git a/crates/openshell-supervisor-process/src/netns/mod.rs b/crates/openshell-supervisor-process/src/netns/mod.rs index cc7b1d84ce..44e9470931 100644 --- a/crates/openshell-supervisor-process/src/netns/mod.rs +++ b/crates/openshell-supervisor-process/src/netns/mod.rs @@ -285,43 +285,22 @@ impl NetworkNamespace { // monitor can see log entries from the sandbox namespace. enable_nf_log_all_netns(); - // Try combined ruleset with log rules first. Log rules must appear - // before reject rules in the chain so packets are logged before being - // rejected. If the kernel lacks nft_log support, fall back to the - // reject-only ruleset. - let ruleset_with_log = - nft_ruleset::generate_bypass_ruleset(&host_ip_str, proxy_port, Some(&log_prefix)); - - if let Err(e) = run_nft_netns(&self.name, &nft_path, &ruleset_with_log) { + let commands = + nft_ruleset::generate_bypass_commands(&host_ip_str, proxy_port, Some(&log_prefix)); + + if let Err(e) = run_nft_commands_netns(&self.name, &nft_path, &commands) { openshell_ocsf::ocsf_emit!( openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(openshell_ocsf::SeverityId::Low) + .severity(openshell_ocsf::SeverityId::Medium) .status(openshell_ocsf::StatusId::Failure) - .state(openshell_ocsf::StateId::Other, "degraded") + .state(openshell_ocsf::StateId::Disabled, "failed") .message(format!( - "Failed to install bypass log rules (non-fatal), falling back to reject-only [ns:{}]: {e}", + "Failed to install bypass detection rules [ns:{}]: {e}", self.name )) .build() ); - - let ruleset_no_log = - nft_ruleset::generate_bypass_ruleset(&host_ip_str, proxy_port, None); - - if let Err(e) = run_nft_netns(&self.name, &nft_path, &ruleset_no_log) { - openshell_ocsf::ocsf_emit!( - openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(openshell_ocsf::SeverityId::Medium) - .status(openshell_ocsf::StatusId::Failure) - .state(openshell_ocsf::StateId::Disabled, "failed") - .message(format!( - "Failed to install bypass detection rules [ns:{}]: {e}", - self.name - )) - .build() - ); - return Err(e); - } + return Err(e); } openshell_ocsf::ocsf_emit!( @@ -467,6 +446,193 @@ pub fn create_netns_for_proxy( } } +/// Install pod-network bypass enforcement for Kubernetes sidecar topology. +/// +/// This runs in the current network namespace, not in a per-workload netns. +/// The rules allow loopback and the sidecar proxy UID, then reject direct +/// TCP/UDP egress from other UIDs so traffic must use the sidecar's local +/// proxy. +/// +/// # Errors +/// +/// Returns an error when `nft` is unavailable or the ruleset cannot be loaded. +pub fn install_sidecar_bypass_rules(proxy_uid: u32) -> Result<()> { + match install_sidecar_nft_bypass_rules(proxy_uid) { + Ok(()) => Ok(()), + Err(nft_error) => { + warn!( + error = %nft_error, + "Failed to install nftables sidecar rules; trying iptables-legacy fallback" + ); + install_sidecar_iptables_legacy_bypass_rules(proxy_uid).map_err(|iptables_error| { + miette::miette!( + "sidecar nft ruleset load failed: {nft_error}; sidecar iptables-legacy fallback failed: {iptables_error}" + ) + }) + } + } +} + +fn install_sidecar_nft_bypass_rules(proxy_uid: u32) -> Result<()> { + let nft_cmd = find_nft().ok_or_else(|| { + miette::miette!( + "trusted nft helper not found; sidecar network enforcement requires nftables" + ) + })?; + let log_prefix = Some("openshell:sidecar-bypass:"); + let commands = nft_ruleset::generate_sidecar_bypass_commands(proxy_uid, log_prefix); + run_nft_commands_current_namespace(&nft_cmd, &commands) +} + +const SIDECAR_IPTABLES_CHAIN: &str = "OPENSHELL_SIDECAR_BYPASS"; +const PROC_NET_IF_INET6_PATH: &str = "/proc/net/if_inet6"; + +fn install_sidecar_iptables_legacy_bypass_rules(proxy_uid: u32) -> Result<()> { + let ipv4_filter_tool = find_iptables_legacy().ok_or_else(|| { + miette::miette!( + "trusted iptables-legacy helper not found; sidecar network enforcement fallback unavailable" + ) + })?; + + let ipv6_fence_tool = if current_namespace_has_non_loopback_ipv6()? { + Some(find_ip6tables_legacy().ok_or_else(|| { + miette::miette!( + "trusted ip6tables-legacy helper not found; sidecar network enforcement fallback cannot fence IPv6" + ) + })?) + } else { + warn!( + "Skipping IPv6 sidecar iptables-legacy fallback because the current namespace has no non-loopback IPv6 interface" + ); + None + }; + + cleanup_sidecar_iptables_legacy_rule_families(&ipv4_filter_tool, ipv6_fence_tool.as_deref()); + + if let Err(e) = install_sidecar_iptables_legacy_family_rules( + &ipv4_filter_tool, + proxy_uid, + "icmp-port-unreachable", + ) { + cleanup_sidecar_iptables_legacy_rule_families( + &ipv4_filter_tool, + ipv6_fence_tool.as_deref(), + ); + return Err(e); + } + + if let Some(ipv6_fence_tool) = ipv6_fence_tool + && let Err(e) = install_sidecar_iptables_legacy_family_rules( + &ipv6_fence_tool, + proxy_uid, + "icmp6-port-unreachable", + ) + { + cleanup_sidecar_iptables_legacy_rule_families(&ipv4_filter_tool, Some(&ipv6_fence_tool)); + return Err(e); + } + + Ok(()) +} + +fn current_namespace_has_non_loopback_ipv6() -> Result { + match std::fs::read_to_string(PROC_NET_IF_INET6_PATH) { + Ok(content) => Ok(has_non_loopback_ipv6_interface(&content)), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(e) => Err(miette::miette!( + "failed to inspect {PROC_NET_IF_INET6_PATH} before installing sidecar IPv6 fence: {e}" + )), + } +} + +fn has_non_loopback_ipv6_interface(content: &str) -> bool { + content.lines().any(|line| { + line.split_whitespace() + .nth(5) + .is_some_and(|iface| iface != "lo") + }) +} + +fn install_sidecar_iptables_legacy_family_rules( + cmd: &str, + proxy_uid: u32, + udp_reject_with: &str, +) -> Result<()> { + let proxy_uid_arg = proxy_uid.to_string(); + let commands: Vec> = vec![ + vec!["-N", SIDECAR_IPTABLES_CHAIN], + vec!["-A", SIDECAR_IPTABLES_CHAIN, "-o", "lo", "-j", "ACCEPT"], + vec![ + "-A", + SIDECAR_IPTABLES_CHAIN, + "-m", + "conntrack", + "--ctstate", + "ESTABLISHED,RELATED", + "-j", + "ACCEPT", + ], + vec![ + "-A", + SIDECAR_IPTABLES_CHAIN, + "-m", + "owner", + "--uid-owner", + &proxy_uid_arg, + "-j", + "ACCEPT", + ], + vec![ + "-A", + SIDECAR_IPTABLES_CHAIN, + "-p", + "tcp", + "-j", + "REJECT", + "--reject-with", + "tcp-reset", + ], + vec![ + "-A", + SIDECAR_IPTABLES_CHAIN, + "-p", + "udp", + "-j", + "REJECT", + "--reject-with", + udp_reject_with, + ], + vec!["-A", "OUTPUT", "-j", SIDECAR_IPTABLES_CHAIN], + ]; + + for args in commands { + if let Err(e) = run_iptables_legacy_current_namespace(cmd, &args) { + cleanup_sidecar_iptables_legacy_rules(cmd); + return Err(e); + } + } + + Ok(()) +} + +fn cleanup_sidecar_iptables_legacy_rules(iptables_cmd: &str) { + while run_iptables_legacy_current_namespace( + iptables_cmd, + &["-D", "OUTPUT", "-j", SIDECAR_IPTABLES_CHAIN], + ) + .is_ok() + {} + let _ = run_iptables_legacy_current_namespace(iptables_cmd, &["-F", SIDECAR_IPTABLES_CHAIN]); + let _ = run_iptables_legacy_current_namespace(iptables_cmd, &["-X", SIDECAR_IPTABLES_CHAIN]); +} + +fn cleanup_sidecar_iptables_legacy_rule_families(ipv4_cmd: &str, ipv6_cmd: Option<&str>) { + cleanup_sidecar_iptables_legacy_rules(ipv4_cmd); + if let Some(ipv6_cmd) = ipv6_cmd { + cleanup_sidecar_iptables_legacy_rules(ipv6_cmd); + } +} + /// Run an `ip` command on the host. fn run_ip(args: &[&str]) -> Result<()> { let ip_path = find_trusted_binary("ip", IP_SEARCH_PATHS)?; @@ -490,6 +656,68 @@ fn run_ip(args: &[&str]) -> Result<()> { Ok(()) } +fn run_iptables_legacy_current_namespace(iptables_cmd: &str, args: &[&str]) -> Result<()> { + debug!( + command = %format!("{iptables_cmd} {}", args.join(" ")), + "Running iptables-legacy sidecar command" + ); + + let output = Command::new(iptables_cmd) + .args(args) + .output() + .into_diagnostic()?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(miette::miette!( + "{iptables_cmd} {} failed: {}", + args.join(" "), + stderr.trim() + )); + } + + Ok(()) +} + +/// Run a sequence of nft commands in the current network namespace. +/// +/// Each command is executed as a separate `nft` invocation to avoid atomic +/// batch rollback (where one unsupported expression like `ct state` or `log` +/// causes the entire transaction, including table creation, to fail). +/// +/// Commands marked as non-required are allowed to fail with a warning. +/// Required commands that fail abort the sequence immediately. +fn run_nft_commands_current_namespace( + nft_cmd: &str, + commands: &[nft_ruleset::NftCommand], +) -> Result<()> { + for cmd in commands { + let args_str = cmd.args.join(" "); + debug!(command = %format!("{nft_cmd} {args_str}"), "Running nft command"); + + let output = Command::new(nft_cmd) + .args(&cmd.args) + .output() + .into_diagnostic()?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + if cmd.required { + return Err(miette::miette!( + "{nft_cmd} {args_str} failed: {}", + stderr.trim() + )); + } + warn!( + command = %args_str, + error = %stderr.trim(), + "non-required nft command failed (continuing)" + ); + } + } + Ok(()) +} + /// Run an `ip` command inside a network namespace via `nsenter --net=`. /// /// We use `nsenter` instead of `ip netns exec` because `ip netns exec` @@ -532,47 +760,51 @@ fn run_ip_netns(netns: &str, args: &[&str]) -> Result<()> { Ok(()) } -/// Load an nftables ruleset inside a network namespace via `nsenter --net=`. +/// Run a sequence of nft commands inside a network namespace via `nsenter --net=`. /// -/// Writes the ruleset to a temp file and loads it with `nft -f `. -/// A temp file is used instead of piping to stdin (`nft -f -`) because -/// `nft` resolves `-` to `/dev/stdin`, which may not exist in minimal -/// VM guest environments (e.g. virtiofs rootfs without /proc mounted -/// at nft invocation time). -fn run_nft_netns(netns: &str, nft_cmd: &str, ruleset: &str) -> Result<()> { - use std::io::Write; - let mut tmp = tempfile::Builder::new() - .prefix("openshell-nft-") - .suffix(".conf") - .tempfile() - .into_diagnostic()?; - tmp.write_all(ruleset.as_bytes()).into_diagnostic()?; - let ruleset_path = tmp.path().to_string_lossy().to_string(); - +/// Each command is executed as a separate invocation to avoid atomic batch +/// rollback. See [`run_nft_commands_current_namespace`] for rationale. +fn run_nft_commands_netns( + netns: &str, + nft_cmd: &str, + commands: &[nft_ruleset::NftCommand], +) -> Result<()> { let nsenter_path = find_trusted_binary("nsenter", NSENTER_SEARCH_PATHS)?; let ns_path = format!("/var/run/netns/{netns}"); let net_flag = format!("--net={ns_path}"); - debug!( - command = %format!("{nsenter_path} {net_flag} -- {nft_cmd} -f {ruleset_path}"), - "Loading nftables ruleset in namespace" - ); + for cmd in commands { + let args_str = cmd.args.join(" "); + debug!( + command = %format!("{nsenter_path} {net_flag} -- {nft_cmd} {args_str}"), + "Running nft command in namespace" + ); - let output = Command::new(nsenter_path) - .args([net_flag.as_str(), "--", nft_cmd, "-f", &ruleset_path]) - .output() - .into_diagnostic()?; + let mut full_args = vec![net_flag.as_str(), "--", nft_cmd]; + let arg_refs: Vec<&str> = cmd.args.iter().map(String::as_str).collect(); + full_args.extend(&arg_refs); - drop(tmp); + let output = Command::new(nsenter_path) + .args(&full_args) + .output() + .into_diagnostic()?; - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(miette::miette!( - "nft ruleset load failed in netns {netns}: {}", - stderr.trim() - )); + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + if cmd.required { + return Err(miette::miette!( + "nft {args_str} failed in netns {netns}: {}", + stderr.trim() + )); + } + warn!( + command = %args_str, + error = %stderr.trim(), + netns = %netns, + "non-required nft command failed in namespace (continuing)" + ); + } } - Ok(()) } @@ -605,6 +837,16 @@ fn enable_nf_log_all_netns() { /// Well-known paths where nft may be installed. const NFT_SEARCH_PATHS: &[&str] = &["/usr/sbin/nft", "/sbin/nft", "/usr/bin/nft"]; +const IPTABLES_LEGACY_SEARCH_PATHS: &[&str] = &[ + "/usr/sbin/iptables-legacy", + "/sbin/iptables-legacy", + "/usr/bin/iptables-legacy", +]; +const IP6TABLES_LEGACY_SEARCH_PATHS: &[&str] = &[ + "/usr/sbin/ip6tables-legacy", + "/sbin/ip6tables-legacy", + "/usr/bin/ip6tables-legacy", +]; fn find_trusted_binary<'a>(name: &str, paths: &'a [&str]) -> Result<&'a str> { paths @@ -629,6 +871,18 @@ fn find_nft() -> Option { .map(String::from) } +fn find_iptables_legacy() -> Option { + find_trusted_binary("iptables-legacy", IPTABLES_LEGACY_SEARCH_PATHS) + .ok() + .map(String::from) +} + +fn find_ip6tables_legacy() -> Option { + find_trusted_binary("ip6tables-legacy", IP6TABLES_LEGACY_SEARCH_PATHS) + .ok() + .map(String::from) +} + #[cfg(test)] mod tests { use super::*; @@ -668,6 +922,49 @@ mod tests { } } + #[test] + fn iptables_legacy_search_paths_are_absolute() { + for path in IPTABLES_LEGACY_SEARCH_PATHS { + assert!( + path.starts_with('/'), + "IPTABLES_LEGACY_SEARCH_PATHS entry must be absolute: {path}" + ); + } + } + + #[test] + fn ip6tables_legacy_search_paths_are_absolute() { + for path in IP6TABLES_LEGACY_SEARCH_PATHS { + assert!( + path.starts_with('/'), + "IP6TABLES_LEGACY_SEARCH_PATHS entry must be absolute: {path}" + ); + } + } + + #[test] + fn non_loopback_ipv6_detector_ignores_empty_input() { + assert!(!has_non_loopback_ipv6_interface("")); + assert!(!has_non_loopback_ipv6_interface("\n\n")); + } + + #[test] + fn non_loopback_ipv6_detector_ignores_loopback() { + let content = "00000000000000000000000000000001 01 80 10 80 lo\n"; + + assert!(!has_non_loopback_ipv6_interface(content)); + } + + #[test] + fn non_loopback_ipv6_detector_detects_pod_interface() { + let content = "\ +00000000000000000000000000000001 01 80 10 80 lo +fe800000000000000000000000000001 02 40 20 80 eth0 +"; + + assert!(has_non_loopback_ipv6_interface(content)); + } + #[test] #[ignore = "requires root privileges"] fn test_create_and_drop_namespace() { diff --git a/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs b/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs index ba7aeb9368..25b4549ab5 100644 --- a/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs +++ b/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs @@ -6,85 +6,417 @@ //! This module provides pure functions to generate nftables rulesets that enforce //! the sandbox network policy: all traffic must go through the proxy, with bypass //! attempts logged and rejected. +//! +//! Rulesets are returned as a sequence of individual nft commands rather than a +//! monolithic file. Running each command as a separate `nft` invocation avoids +//! `nft -f` atomic batch semantics, where a single unsupported expression (e.g. +//! `ct state` without `nf_conntrack`, `log` without `nf_log`) rolls back the +//! entire transaction including table/chain creation. + +/// A single nft command with metadata about whether it is required. +pub struct NftCommand { + /// The nft command arguments (e.g. `["add", "table", "inet", "openshell_bypass"]`). + pub args: Vec, + /// When false, failure of this command is non-fatal; the caller should + /// log a warning and continue with the remaining commands. + pub required: bool, +} -/// Generate a complete nftables ruleset for sandbox network bypass enforcement. +/// Generate nft commands for sandbox network bypass enforcement. /// /// Creates an `inet` family table (handles both IPv4 and IPv6) with rules that: /// 1. Accept traffic to the proxy (IPv4 only) /// 2. Accept loopback traffic -/// 3. Accept established/related connections +/// 3. Accept established/related connections (optional; requires `nf_conntrack`) /// 4. Reject TCP and UDP bypass attempts (both IPv4 and IPv6) /// /// If `log_prefix` is provided, log rules are inserted before each reject rule /// so that bypass attempts are recorded in the kernel ring buffer before being -/// rejected. The `log` expression requires kernel `nft_log` module support; -/// pass `None` for `log_prefix` as a fallback when that module is unavailable. -pub fn generate_bypass_ruleset(host_ip: &str, proxy_port: u16, log_prefix: Option<&str>) -> String { - let log_tcp = log_prefix - .map(|p| { - format!( - "\n tcp flags syn limit rate 5/second burst 10 packets log prefix \"{p}\" flags skuid" - ) - }) - .unwrap_or_default(); - let log_udp = log_prefix - .map(|p| { - format!( - "\n meta l4proto udp limit rate 5/second burst 10 packets log prefix \"{p}\" flags skuid" - ) - }) - .unwrap_or_default(); - - format!( - r#"table inet openshell_bypass {{ - chain output {{ - type filter hook output priority 0; policy accept; - - ip daddr {host_ip} tcp dport {proxy_port} accept - oifname "lo" accept - ct state established,related accept{log_tcp} - meta nfproto ipv4 meta l4proto tcp reject with icmp type port-unreachable - meta nfproto ipv6 meta l4proto tcp reject with icmpv6 type port-unreachable{log_udp} - meta nfproto ipv4 meta l4proto udp reject with icmp type port-unreachable - meta nfproto ipv6 meta l4proto udp reject with icmpv6 type port-unreachable - }} -}} -"# - ) +/// rejected. Log rules are always non-required since they need `nf_log` support. +pub fn generate_bypass_commands( + host_ip: &str, + proxy_port: u16, + log_prefix: Option<&str>, +) -> Vec { + let table = "openshell_bypass"; + let mut cmds = vec![ + nft_cmd(true, &["add", "table", "inet", table]), + nft_cmd(true, &["flush", "table", "inet", table]), + nft_cmd( + true, + &[ + "add", + "chain", + "inet", + table, + "output", + "{ type filter hook output priority 0; policy accept; }", + ], + ), + nft_cmd( + true, + &[ + "add", + "rule", + "inet", + table, + "output", + "ip", + "daddr", + host_ip, + "tcp", + "dport", + &proxy_port.to_string(), + "accept", + ], + ), + nft_cmd( + true, + &[ + "add", "rule", "inet", table, "output", "oifname", "lo", "accept", + ], + ), + nft_cmd( + false, + &[ + "add", + "rule", + "inet", + table, + "output", + "ct", + "state", + "established,related", + "accept", + ], + ), + ]; + + if let Some(prefix) = log_prefix { + cmds.push(nft_cmd( + false, + &[ + "add", "rule", "inet", table, "output", "tcp", "flags", "syn", "limit", "rate", + "5/second", "burst", "10", "packets", "log", "prefix", prefix, "flags", "skuid", + ], + )); + } + + cmds.push(nft_cmd( + true, + &[ + "add", + "rule", + "inet", + table, + "output", + "meta", + "nfproto", + "ipv4", + "meta", + "l4proto", + "tcp", + "reject", + "with", + "icmp", + "type", + "port-unreachable", + ], + )); + cmds.push(nft_cmd( + true, + &[ + "add", + "rule", + "inet", + table, + "output", + "meta", + "nfproto", + "ipv6", + "meta", + "l4proto", + "tcp", + "reject", + "with", + "icmpv6", + "type", + "port-unreachable", + ], + )); + + if let Some(prefix) = log_prefix { + cmds.push(nft_cmd( + false, + &[ + "add", "rule", "inet", table, "output", "meta", "l4proto", "udp", "limit", "rate", + "5/second", "burst", "10", "packets", "log", "prefix", prefix, "flags", "skuid", + ], + )); + } + + cmds.push(nft_cmd( + true, + &[ + "add", + "rule", + "inet", + table, + "output", + "meta", + "nfproto", + "ipv4", + "meta", + "l4proto", + "udp", + "reject", + "with", + "icmp", + "type", + "port-unreachable", + ], + )); + cmds.push(nft_cmd( + true, + &[ + "add", + "rule", + "inet", + table, + "output", + "meta", + "nfproto", + "ipv6", + "meta", + "l4proto", + "udp", + "reject", + "with", + "icmpv6", + "type", + "port-unreachable", + ], + )); + + cmds +} + +/// Generate nft commands for Kubernetes sidecar enforcement. +/// +/// The network sidecar and the process supervisor share a pod network +/// namespace. The sidecar runs as `proxy_uid` and owns external egress; +/// sandbox traffic must use loopback services hosted by that sidecar +/// (gateway forward and HTTP CONNECT proxy). The generated fence rejects +/// TCP/UDP bypass attempts from non-proxy UIDs; other L4 protocols are outside +/// the sidecar policy fence. +pub fn generate_sidecar_bypass_commands( + proxy_uid: u32, + log_prefix: Option<&str>, +) -> Vec { + let table = "openshell_sidecar_bypass"; + let uid_str = proxy_uid.to_string(); + let mut cmds = vec![ + nft_cmd(true, &["add", "table", "inet", table]), + nft_cmd(true, &["flush", "table", "inet", table]), + nft_cmd( + true, + &[ + "add", + "chain", + "inet", + table, + "output", + "{ type filter hook output priority 0; policy accept; }", + ], + ), + nft_cmd( + true, + &[ + "add", "rule", "inet", table, "output", "oifname", "lo", "accept", + ], + ), + nft_cmd( + false, + &[ + "add", + "rule", + "inet", + table, + "output", + "ct", + "state", + "established,related", + "accept", + ], + ), + nft_cmd( + true, + &[ + "add", "rule", "inet", table, "output", "meta", "skuid", &uid_str, "accept", + ], + ), + ]; + + if let Some(prefix) = log_prefix { + cmds.push(nft_cmd( + false, + &[ + "add", "rule", "inet", table, "output", "tcp", "flags", "syn", "limit", "rate", + "5/second", "burst", "10", "packets", "log", "prefix", prefix, "flags", "skuid", + ], + )); + } + + cmds.push(nft_cmd( + true, + &[ + "add", + "rule", + "inet", + table, + "output", + "meta", + "nfproto", + "ipv4", + "meta", + "l4proto", + "tcp", + "reject", + "with", + "icmp", + "type", + "port-unreachable", + ], + )); + cmds.push(nft_cmd( + true, + &[ + "add", + "rule", + "inet", + table, + "output", + "meta", + "nfproto", + "ipv6", + "meta", + "l4proto", + "tcp", + "reject", + "with", + "icmpv6", + "type", + "port-unreachable", + ], + )); + + if let Some(prefix) = log_prefix { + cmds.push(nft_cmd( + false, + &[ + "add", "rule", "inet", table, "output", "meta", "l4proto", "udp", "limit", "rate", + "5/second", "burst", "10", "packets", "log", "prefix", prefix, "flags", "skuid", + ], + )); + } + + cmds.push(nft_cmd( + true, + &[ + "add", + "rule", + "inet", + table, + "output", + "meta", + "nfproto", + "ipv4", + "meta", + "l4proto", + "udp", + "reject", + "with", + "icmp", + "type", + "port-unreachable", + ], + )); + cmds.push(nft_cmd( + true, + &[ + "add", + "rule", + "inet", + table, + "output", + "meta", + "nfproto", + "ipv6", + "meta", + "l4proto", + "udp", + "reject", + "with", + "icmpv6", + "type", + "port-unreachable", + ], + )); + + cmds +} + +fn nft_cmd(required: bool, args: &[&str]) -> NftCommand { + NftCommand { + args: args.iter().map(|s| (*s).to_string()).collect(), + required, + } } #[cfg(test)] mod tests { use super::*; + fn cmd_str(cmd: &NftCommand) -> String { + cmd.args.join(" ") + } + + fn all_strs(cmds: &[NftCommand]) -> String { + cmds.iter().map(cmd_str).collect::>().join("\n") + } + #[test] - fn generates_bypass_ruleset_with_proxy_rule() { - let ruleset = generate_bypass_ruleset("10.0.2.2", 8080, None); - assert!(ruleset.contains("table inet openshell_bypass")); - assert!(ruleset.contains("chain output")); - assert!(ruleset.contains("ip daddr 10.0.2.2 tcp dport 8080 accept")); + fn generates_bypass_commands_with_proxy_rule() { + let cmds = generate_bypass_commands("10.0.2.2", 8080, None); + let text = all_strs(&cmds); + assert!(text.contains("add table inet openshell_bypass")); + assert!(text.contains("add chain inet openshell_bypass output")); + assert!(text.contains("ip daddr 10.0.2.2 tcp dport 8080 accept")); } #[test] - fn ruleset_has_inet_family_table_and_output_chain() { - let ruleset = generate_bypass_ruleset("192.168.1.1", 3128, None); - assert!(ruleset.contains("table inet openshell_bypass")); - assert!(ruleset.contains("type filter hook output priority 0; policy accept;")); + fn bypass_commands_have_table_and_chain() { + let cmds = generate_bypass_commands("192.168.1.1", 3128, None); + let text = all_strs(&cmds); + assert!(text.contains("add table inet openshell_bypass")); + assert!(text.contains("type filter hook output priority 0; policy accept;")); } #[test] fn proxy_accept_rule_uses_provided_ip_and_port() { - let ruleset = generate_bypass_ruleset("172.16.0.1", 9999, None); - assert!(ruleset.contains("ip daddr 172.16.0.1 tcp dport 9999 accept")); + let cmds = generate_bypass_commands("172.16.0.1", 9999, None); + let text = all_strs(&cmds); + assert!(text.contains("ip daddr 172.16.0.1 tcp dport 9999 accept")); } #[test] fn rules_are_ordered_accept_then_reject() { - let ruleset = generate_bypass_ruleset("10.0.2.2", 8080, None); - let proxy_pos = ruleset.find("ip daddr").unwrap(); - let lo_pos = ruleset.find("oifname \"lo\"").unwrap(); - let ct_pos = ruleset.find("ct state established,related").unwrap(); - let reject_pos = ruleset.find("reject with icmp type").unwrap(); + let cmds = generate_bypass_commands("10.0.2.2", 8080, None); + let text = all_strs(&cmds); + let proxy_pos = text.find("ip daddr").unwrap(); + let lo_pos = text.find("oifname lo").unwrap(); + let ct_pos = text.find("ct state established,related").unwrap(); + let reject_pos = text.find("reject with icmp type").unwrap(); assert!(proxy_pos < lo_pos); assert!(lo_pos < ct_pos); @@ -93,11 +425,12 @@ mod tests { #[test] fn both_ipv4_and_ipv6_reject_types_are_present() { - let ruleset = generate_bypass_ruleset("10.0.2.2", 8080, None); - let icmp_count = ruleset + let cmds = generate_bypass_commands("10.0.2.2", 8080, None); + let text = all_strs(&cmds); + let icmp_count = text .matches("reject with icmp type port-unreachable") .count(); - let icmpv6_count = ruleset + let icmpv6_count = text .matches("reject with icmpv6 type port-unreachable") .count(); assert_eq!(icmp_count, 2, "need IPv4 ICMP rejects for TCP + UDP"); @@ -105,34 +438,35 @@ mod tests { } #[test] - fn no_log_ruleset_omits_log_rules() { - let ruleset = generate_bypass_ruleset("10.0.2.2", 8080, None); + fn no_log_commands_omit_log_rules() { + let cmds = generate_bypass_commands("10.0.2.2", 8080, None); + let text = all_strs(&cmds); assert!( - !ruleset.contains("log prefix"), - "no-log ruleset must not contain log rules" + !text.contains("log prefix"), + "no-log commands must not contain log rules" ); } #[test] - fn log_ruleset_contains_prefix_for_tcp_and_udp() { - let ruleset = generate_bypass_ruleset("10.0.2.2", 8080, Some("openshell:bypass:test:")); - let count = ruleset - .matches("log prefix \"openshell:bypass:test:\"") - .count(); + fn log_commands_contain_prefix_for_tcp_and_udp() { + let cmds = generate_bypass_commands("10.0.2.2", 8080, Some("openshell:bypass:test:")); + let text = all_strs(&cmds); + let count = text.matches("log prefix openshell:bypass:test:").count(); assert_eq!(count, 2, "need log rules for both TCP and UDP"); - assert!(ruleset.contains("tcp flags syn limit rate 5/second burst 10 packets")); - assert!(ruleset.contains("meta l4proto udp limit rate 5/second burst 10 packets")); + assert!(text.contains("tcp flags syn limit rate 5/second burst 10 packets")); + assert!(text.contains("meta l4proto udp limit rate 5/second burst 10 packets")); } #[test] fn log_rules_appear_before_reject_rules() { - let ruleset = generate_bypass_ruleset("10.0.2.2", 8080, Some("openshell:bypass:test:")); - let tcp_log_pos = ruleset.find("tcp flags syn").unwrap(); - let tcp_reject_pos = ruleset + let cmds = generate_bypass_commands("10.0.2.2", 8080, Some("openshell:bypass:test:")); + let text = all_strs(&cmds); + let tcp_log_pos = text.find("tcp flags syn").unwrap(); + let tcp_reject_pos = text .find("meta nfproto ipv4 meta l4proto tcp reject") .unwrap(); - let udp_log_pos = ruleset.find("meta l4proto udp limit rate").unwrap(); - let udp_reject_pos = ruleset + let udp_log_pos = text.find("meta l4proto udp limit rate").unwrap(); + let udp_reject_pos = text .find("meta nfproto ipv4 meta l4proto udp reject") .unwrap(); @@ -145,4 +479,53 @@ mod tests { "UDP log rule must come before UDP reject rule" ); } + + #[test] + fn ct_state_rule_is_not_required() { + let cmds = generate_bypass_commands("10.0.2.2", 8080, None); + let ct_cmd = cmds + .iter() + .find(|c| cmd_str(c).contains("ct state")) + .unwrap(); + assert!( + !ct_cmd.required, + "ct state rule should be non-required (needs nf_conntrack)" + ); + } + + #[test] + fn log_rules_are_not_required() { + let cmds = generate_bypass_commands("10.0.2.2", 8080, Some("openshell:bypass:test:")); + for cmd in &cmds { + if cmd_str(cmd).contains("log prefix") { + assert!( + !cmd.required, + "log rules should be non-required (needs nf_log)" + ); + } + } + } + + #[test] + fn sidecar_commands_allow_supervisor_uid_and_loopback() { + let cmds = generate_sidecar_bypass_commands(1337, None); + let text = all_strs(&cmds); + assert!(text.contains("add table inet openshell_sidecar_bypass")); + assert!(text.contains("oifname lo accept")); + assert!(text.contains("meta skuid 1337 accept")); + } + + #[test] + fn sidecar_commands_reject_tcp_and_udp_egress() { + let cmds = generate_sidecar_bypass_commands(0, Some("openshell:sidecar:test:")); + let text = all_strs(&cmds); + assert!(text.contains("meta nfproto ipv4 meta l4proto tcp reject")); + assert!(text.contains("meta nfproto ipv6 meta l4proto tcp reject")); + assert!(text.contains("meta nfproto ipv4 meta l4proto udp reject")); + assert!(text.contains("meta nfproto ipv6 meta l4proto udp reject")); + assert_eq!( + text.matches("log prefix openshell:sidecar:test:").count(), + 2 + ); + } } diff --git a/crates/openshell-supervisor-process/src/process.rs b/crates/openshell-supervisor-process/src/process.rs index fcd7ae69c1..72effa98f8 100644 --- a/crates/openshell-supervisor-process/src/process.rs +++ b/crates/openshell-supervisor-process/src/process.rs @@ -28,10 +28,55 @@ use std::sync::OnceLock; use tokio::process::{Child, Command}; use tracing::{debug, info}; +/// Process/filesystem enforcement performed by the process supervisor. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProcessEnforcementMode { + /// Preserve the existing supervisor behavior: prepare filesystem policy, + /// drop privileges, and apply Landlock/seccomp to workload processes. + Full, + /// Preserve process launch and SSH/session behavior, but skip controls + /// that require root or extra Linux capabilities. Kubernetes sidecar mode + /// uses this when network policy is enforced by the network sidecar. + NetworkOnly, +} + +impl ProcessEnforcementMode { + #[must_use] + pub const fn uses_privileged_process_setup(self) -> bool { + matches!(self, Self::Full) + } + + #[must_use] + pub const fn enforces_child_sandbox(self) -> bool { + matches!(self, Self::Full | Self::NetworkOnly) + } +} + +#[cfg(target_os = "linux")] +pub(crate) fn prepare_child_sandbox( + policy: &SandboxPolicy, + workdir: Option<&str>, + enforcement_mode: ProcessEnforcementMode, +) -> Result> { + if !enforcement_mode.enforces_child_sandbox() { + return Ok(None); + } + + let prepared = if enforcement_mode.uses_privileged_process_setup() { + sandbox::linux::prepare(policy, workdir) + } else { + sandbox::linux::prepare_current_user(policy, workdir) + }?; + Ok(Some(prepared)) +} + const SUPERVISOR_ONLY_ENV_VARS: &[&str] = &[ openshell_core::sandbox_env::SANDBOX_TOKEN, openshell_core::sandbox_env::SANDBOX_TOKEN_FILE, openshell_core::sandbox_env::K8S_SA_TOKEN_FILE, + openshell_core::sandbox_env::TLS_CA, + openshell_core::sandbox_env::TLS_CERT, + openshell_core::sandbox_env::TLS_KEY, openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, ]; @@ -443,6 +488,7 @@ impl ProcessHandle { workdir: Option<&str>, interactive: bool, policy: &SandboxPolicy, + enforcement_mode: ProcessEnforcementMode, netns: Option<&NetworkNamespace>, ca_paths: Option<&(PathBuf, PathBuf)>, provider_env: &HashMap, @@ -453,6 +499,7 @@ impl ProcessHandle { workdir, interactive, policy, + enforcement_mode, netns.and_then(NetworkNamespace::ns_fd), ca_paths, provider_env, @@ -465,12 +512,14 @@ impl ProcessHandle { /// /// Returns an error if the process fails to start. #[cfg(not(target_os = "linux"))] + #[allow(clippy::too_many_arguments)] pub fn spawn( program: &str, args: &[String], workdir: Option<&str>, interactive: bool, policy: &SandboxPolicy, + enforcement_mode: ProcessEnforcementMode, ca_paths: Option<&(PathBuf, PathBuf)>, provider_env: &HashMap, ) -> Result { @@ -480,6 +529,7 @@ impl ProcessHandle { workdir, interactive, policy, + enforcement_mode, ca_paths, provider_env, ) @@ -493,6 +543,7 @@ impl ProcessHandle { workdir: Option<&str>, interactive: bool, policy: &SandboxPolicy, + enforcement_mode: ProcessEnforcementMode, netns_fd: Option, ca_paths: Option<&(PathBuf, PathBuf)>, provider_env: &HashMap, @@ -552,18 +603,26 @@ impl ProcessHandle { // process where the tracing subscriber is functional. The child's // pre_exec context cannot reliably emit structured logs. #[cfg(target_os = "linux")] - sandbox::linux::log_sandbox_readiness(policy, workdir); + if enforcement_mode.enforces_child_sandbox() { + sandbox::linux::log_sandbox_readiness(policy, workdir); + } - // Phase 1 (as root): Prepare Landlock ruleset by opening PathFds. - // This MUST happen before drop_privileges() so that root-only paths - // (e.g. mode 700 directories) can be opened. See issue #803. + // Phase 1: Prepare Landlock ruleset by opening PathFds. + // In full mode this runs before drop_privileges() so root-only paths + // can be opened. In sidecar network-only mode the container already + // runs as the sandbox UID, so inaccessible paths are unavailable to + // the workload and best-effort compatibility skips them. #[cfg(target_os = "linux")] - let prepared_sandbox = sandbox::linux::prepare(policy, workdir) + let prepared_sandbox = prepare_child_sandbox(policy, workdir, enforcement_mode) .map_err(|err| miette::miette!("Failed to prepare sandbox: {err}"))?; #[cfg(target_os = "linux")] - let supervisor_identity_mount = supervisor_identity_mount_from_env().map_err(|err| { - miette::miette!("Failed to prepare supervisor identity isolation: {err}") - })?; + let supervisor_identity_mount = if enforcement_mode.uses_privileged_process_setup() { + supervisor_identity_mount_from_env().map_err(|err| { + miette::miette!("Failed to prepare supervisor identity isolation: {err}") + })? + } else { + None + }; // Set up process group for signal handling (non-interactive mode only). // In interactive mode, we inherit the parent's process group to maintain @@ -575,7 +634,7 @@ impl ProcessHandle { // Wrap in Option so we can .take() it out of the FnMut closure. // pre_exec is only called once (after fork, before exec). #[cfg(target_os = "linux")] - let mut prepared_sandbox = Some(prepared_sandbox); + let mut prepared_sandbox = prepared_sandbox; #[allow(unsafe_code)] unsafe { cmd.pre_exec(move || { @@ -600,8 +659,10 @@ impl ProcessHandle { // Drop privileges. initgroups/setgid/setuid need access to // /etc/group and /etc/passwd which would be blocked if // Landlock were already enforced. - drop_privileges(&policy) - .map_err(|err| std::io::Error::other(err.to_string()))?; + if enforcement_mode.uses_privileged_process_setup() { + drop_privileges(&policy) + .map_err(|err| std::io::Error::other(err.to_string()))?; + } harden_child_process().map_err(|err| std::io::Error::other(err.to_string()))?; @@ -629,12 +690,14 @@ impl ProcessHandle { } #[cfg(not(target_os = "linux"))] + #[allow(clippy::too_many_arguments)] fn spawn_impl( program: &str, args: &[String], workdir: Option<&str>, interactive: bool, policy: &SandboxPolicy, + enforcement_mode: ProcessEnforcementMode, ca_paths: Option<&(PathBuf, PathBuf)>, provider_env: &HashMap, ) -> Result { @@ -697,13 +760,17 @@ impl ProcessHandle { // Drop privileges before applying sandbox restrictions. // initgroups/setgid/setuid need access to /etc/group and /etc/passwd // which may be blocked by Landlock. - drop_privileges(&policy) - .map_err(|err| std::io::Error::other(err.to_string()))?; + if enforcement_mode.uses_privileged_process_setup() { + drop_privileges(&policy) + .map_err(|err| std::io::Error::other(err.to_string()))?; + } harden_child_process().map_err(|err| std::io::Error::other(err.to_string()))?; - sandbox::apply(&policy, workdir.as_deref()) - .map_err(|err| std::io::Error::other(err.to_string()))?; + if enforcement_mode.enforces_child_sandbox() { + sandbox::apply(&policy, workdir.as_deref()) + .map_err(|err| std::io::Error::other(err.to_string()))?; + } Ok(()) }); @@ -1456,6 +1523,18 @@ mod tests { ); } + #[test] + fn full_enforcement_uses_privileged_setup_and_child_sandbox() { + assert!(ProcessEnforcementMode::Full.uses_privileged_process_setup()); + assert!(ProcessEnforcementMode::Full.enforces_child_sandbox()); + } + + #[test] + fn network_only_enforcement_keeps_child_sandbox_without_privileged_setup() { + assert!(!ProcessEnforcementMode::NetworkOnly.uses_privileged_process_setup()); + assert!(ProcessEnforcementMode::NetworkOnly.enforces_child_sandbox()); + } + #[cfg(target_os = "linux")] fn capability_bounding_set_clear_available() -> bool { capctl::caps::CapState::get_current() diff --git a/crates/openshell-supervisor-process/src/run.rs b/crates/openshell-supervisor-process/src/run.rs index e16f118922..5a06d97a8f 100644 --- a/crates/openshell-supervisor-process/src/run.rs +++ b/crates/openshell-supervisor-process/src/run.rs @@ -33,7 +33,7 @@ use openshell_core::denial::DenialEvent; #[cfg(target_os = "linux")] use crate::managed_children; -use crate::process::ProcessHandle; +use crate::process::{ProcessEnforcementMode, ProcessHandle}; fn ocsf_ctx() -> &'static openshell_ocsf::SandboxContext { openshell_ocsf::ctx::ctx() @@ -56,8 +56,11 @@ pub async fn run_process( sandbox_id: Option<&str>, openshell_endpoint: Option<&str>, ssh_socket_path: Option, + shared_ssh_socket: bool, policy: &SandboxPolicy, + enforcement_mode: ProcessEnforcementMode, entrypoint_pid: Arc, + entrypoint_started_tx: Option>, provider_credentials: ProviderCredentialState, provider_env: std::collections::HashMap, ca_file_paths: Option<(std::path::PathBuf, std::path::PathBuf)>, @@ -71,21 +74,26 @@ pub async fn run_process( // /etc/group so the "sandbox" entry matches. Must run before // validate_sandbox_user so passwd lookups see the correct identity. #[cfg(unix)] - crate::process::update_sandbox_passwd_entries()?; + if enforcement_mode.uses_privileged_process_setup() { + crate::process::update_sandbox_passwd_entries()?; + } // Validate that the sandbox user exists in the image. All sandbox images // must include a "sandbox" user for privilege dropping; failing fast here // beats silently running children as root. #[cfg(unix)] - crate::process::validate_sandbox_user(policy)?; - #[cfg(unix)] - crate::process::validate_sandbox_group(policy)?; + if enforcement_mode.uses_privileged_process_setup() { + crate::process::validate_sandbox_user(policy)?; + crate::process::validate_sandbox_group(policy)?; + } // Create read_write directories and chown newly-created ones to the // sandbox user/group. Runs as the supervisor (root) before the child // is forked so the workload sees writable paths it owns. #[cfg(unix)] - crate::process::prepare_filesystem(policy)?; + if enforcement_mode.uses_privileged_process_setup() { + crate::process::prepare_filesystem(policy)?; + } // Eagerly fetch initial settings and install the agent skill if the // proposals flag is on at startup, rather than waiting for the policy @@ -206,31 +214,10 @@ pub async fn run_process( // their env so cooperative tools (curl, npm, Node) route through the // CONNECT proxy. Linux uses the netns host_ip; on other targets fall back // to the policy-declared http_addr directly. - let ssh_proxy_url = if matches!(policy.network.mode, NetworkMode::Proxy) { - #[cfg(target_os = "linux")] - { - netns.map(|ns| { - let port = policy - .network - .proxy - .as_ref() - .and_then(|p| p.http_addr) - .map_or(3128, |addr| addr.port()); - format!("http://{}:{port}", ns.host_ip()) - }) - } - #[cfg(not(target_os = "linux"))] - { - policy - .network - .proxy - .as_ref() - .and_then(|p| p.http_addr) - .map(|addr| format!("http://{addr}")) - } - } else { - None - }; + #[cfg(target_os = "linux")] + let ssh_proxy_url = ssh_proxy_url_for_policy(policy, netns.map(NetworkNamespace::host_ip)); + #[cfg(not(target_os = "linux"))] + let ssh_proxy_url = ssh_proxy_url_for_policy(policy, None); let ssh_socket_path: Option = ssh_socket_path.map(std::path::PathBuf::from); if let Some(listen_path) = ssh_socket_path.clone() { @@ -259,6 +246,8 @@ pub async fn run_process( ca_paths, provider_credentials_clone, user_env_clone, + enforcement_mode, + shared_ssh_socket, ) .await { @@ -314,6 +303,7 @@ pub async fn run_process( id.to_string(), socket.clone(), ssh_netns_fd, + None, ); info!("supervisor session task spawned"); } @@ -325,6 +315,7 @@ pub async fn run_process( workdir, interactive, policy, + enforcement_mode, netns, ca_file_paths.as_ref(), &provider_env, @@ -337,12 +328,16 @@ pub async fn run_process( workdir, interactive, policy, + enforcement_mode, ca_file_paths.as_ref(), &provider_env, )?; // Store the entrypoint PID so the proxy can resolve TCP peer identity entrypoint_pid.store(handle.pid(), Ordering::Release); + if let Some(tx) = entrypoint_started_tx { + let _ = tx.send(handle.pid()); + } ocsf_emit!( ProcessActivityBuilder::new(ocsf_ctx()) .activity(ActivityId::Open) @@ -395,6 +390,23 @@ pub async fn run_process( Ok(status.code()) } +fn ssh_proxy_url_for_policy( + policy: &SandboxPolicy, + netns_proxy_host: Option, +) -> Option { + if !matches!(policy.network.mode, NetworkMode::Proxy) { + return None; + } + + let proxy = policy.network.proxy.as_ref()?; + if let Some(host) = netns_proxy_host { + let port = proxy.http_addr.map_or(3128, |addr| addr.port()); + return Some(format!("http://{host}:{port}")); + } + + proxy.http_addr.map(|addr| format!("http://{addr}")) +} + /// Eagerly fetch initial settings and install the agent-driven policy /// proposal skill if the flag is on at startup. /// @@ -451,3 +463,53 @@ async fn install_initial_agent_skill(sandbox_id: Option<&str>, openshell_endpoin ); } } + +#[cfg(test)] +mod tests { + use super::*; + use openshell_core::policy::{ + FilesystemPolicy, LandlockPolicy, NetworkMode, NetworkPolicy, ProcessPolicy, ProxyPolicy, + }; + + fn policy(mode: NetworkMode, http_addr: Option) -> SandboxPolicy { + SandboxPolicy { + version: 1, + filesystem: FilesystemPolicy::default(), + network: NetworkPolicy { + mode, + proxy: http_addr.map(|http_addr| ProxyPolicy { + http_addr: Some(http_addr), + }), + }, + landlock: LandlockPolicy::default(), + process: ProcessPolicy::default(), + } + } + + #[test] + fn ssh_proxy_url_uses_policy_addr_without_netns() { + let policy = policy(NetworkMode::Proxy, Some(([127, 0, 0, 1], 3128).into())); + + assert_eq!( + ssh_proxy_url_for_policy(&policy, None).as_deref(), + Some("http://127.0.0.1:3128") + ); + } + + #[test] + fn ssh_proxy_url_prefers_netns_host_with_policy_port() { + let policy = policy(NetworkMode::Proxy, Some(([127, 0, 0, 1], 8080).into())); + + assert_eq!( + ssh_proxy_url_for_policy(&policy, Some([10, 200, 0, 1].into())).as_deref(), + Some("http://10.200.0.1:8080") + ); + } + + #[test] + fn ssh_proxy_url_skips_non_proxy_mode() { + let policy = policy(NetworkMode::Allow, Some(([127, 0, 0, 1], 3128).into())); + + assert_eq!(ssh_proxy_url_for_policy(&policy, None), None); + } +} diff --git a/crates/openshell-supervisor-process/src/sandbox/linux/landlock.rs b/crates/openshell-supervisor-process/src/sandbox/linux/landlock.rs index 8808a1a87a..fb82cf02fd 100644 --- a/crates/openshell-supervisor-process/src/sandbox/linux/landlock.rs +++ b/crates/openshell-supervisor-process/src/sandbox/linux/landlock.rs @@ -95,6 +95,12 @@ pub struct PreparedRuleset { compatibility: LandlockCompatibility, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum PathOpenMode { + Privileged, + CurrentUser, +} + /// Phase 1: Open `PathFds` and build the Landlock ruleset **as root**. /// /// This must run before `drop_privileges()` so that `PathFd::new()` can open @@ -103,6 +109,27 @@ pub struct PreparedRuleset { /// Returns `None` if there are no filesystem paths to restrict (no-op). /// Returns `Some(PreparedRuleset)` on success, or an error. pub fn prepare(policy: &SandboxPolicy, workdir: Option<&str>) -> Result> { + prepare_with_path_open_mode(policy, workdir, PathOpenMode::Privileged) +} + +/// Phase 1 for already-unprivileged workloads. +/// +/// Kubernetes sidecar mode starts the process supervisor as the sandbox UID, so +/// Landlock path FDs are opened as the same UID that will run the workload. +/// Paths this UID cannot open are already unavailable to the workload; omit +/// them from the allowlist and let the resulting ruleset deny everything else. +pub fn prepare_current_user( + policy: &SandboxPolicy, + workdir: Option<&str>, +) -> Result> { + prepare_with_path_open_mode(policy, workdir, PathOpenMode::CurrentUser) +} + +fn prepare_with_path_open_mode( + policy: &SandboxPolicy, + workdir: Option<&str>, + path_open_mode: PathOpenMode, +) -> Result> { let read_only = policy.filesystem.read_only.clone(); let mut read_write = policy.filesystem.read_write.clone(); @@ -188,7 +215,7 @@ pub fn prepare(policy: &SandboxPolicy, workdir: Option<&str>) -> Result) -> Result) -> Result<()> { /// /// In `HardRequirement` mode, any failure is fatal — the caller propagates the /// error, which ultimately aborts sandbox startup. -fn try_open_path(path: &Path, compatibility: &LandlockCompatibility) -> Result> { +fn try_open_path( + path: &Path, + compatibility: &LandlockCompatibility, + path_open_mode: PathOpenMode, +) -> Result> { match PathFd::new(path) { Ok(fd) => Ok(Some(fd)), Err(err) => { @@ -335,6 +366,28 @@ fn try_open_path(path: &Path, compatibility: &LandlockCompatibility) -> Result { // NotFound is expected for stale baseline paths (e.g. @@ -421,6 +474,7 @@ mod tests { let result = try_open_path( &PathBuf::from("/nonexistent/openshell/test/path"), &LandlockCompatibility::BestEffort, + PathOpenMode::Privileged, ); assert!(result.is_ok()); assert!(result.unwrap().is_none()); @@ -431,6 +485,7 @@ mod tests { let result = try_open_path( &PathBuf::from("/nonexistent/openshell/test/path"), &LandlockCompatibility::HardRequirement, + PathOpenMode::Privileged, ); assert!(result.is_err()); let err_msg = result.unwrap_err().to_string(); @@ -447,11 +502,26 @@ mod tests { #[test] fn try_open_path_succeeds_for_existing_path() { let dir = tempfile::tempdir().unwrap(); - let result = try_open_path(dir.path(), &LandlockCompatibility::BestEffort); + let result = try_open_path( + dir.path(), + &LandlockCompatibility::BestEffort, + PathOpenMode::Privileged, + ); assert!(result.is_ok()); assert!(result.unwrap().is_some()); } + #[test] + fn try_open_path_current_user_skips_missing_path_in_hard_requirement() { + let result = try_open_path( + &PathBuf::from("/nonexistent/openshell/test/path"), + &LandlockCompatibility::HardRequirement, + PathOpenMode::CurrentUser, + ); + assert!(result.is_ok()); + assert!(result.unwrap().is_none()); + } + #[test] fn classify_not_found() { let err = std::io::Error::from_raw_os_error(libc::ENOENT); diff --git a/crates/openshell-supervisor-process/src/sandbox/linux/mod.rs b/crates/openshell-supervisor-process/src/sandbox/linux/mod.rs index b5397ef079..107a50e370 100644 --- a/crates/openshell-supervisor-process/src/sandbox/linux/mod.rs +++ b/crates/openshell-supervisor-process/src/sandbox/linux/mod.rs @@ -12,7 +12,7 @@ use std::path::PathBuf; use std::sync::Once; /// Opaque handle to a prepared-but-not-yet-enforced sandbox. -/// Holds the Landlock ruleset with `PathFds` opened as root. +/// Holds the Landlock ruleset with `PathFds` opened before child exec. pub struct PreparedSandbox { landlock: Option, policy: SandboxPolicy, @@ -30,6 +30,21 @@ pub fn prepare(policy: &SandboxPolicy, workdir: Option<&str>) -> Result, +) -> Result { + let landlock = landlock::prepare_current_user(policy, workdir)?; + Ok(PreparedSandbox { + landlock, + policy: policy.clone(), + }) +} + /// Phase 2: Enforce prepared sandbox restrictions (after `drop_privileges`). /// /// Calls `restrict_self()` for Landlock and applies seccomp filters. diff --git a/crates/openshell-supervisor-process/src/ssh.rs b/crates/openshell-supervisor-process/src/ssh.rs index 62d10f374d..f5a3ee0793 100644 --- a/crates/openshell-supervisor-process/src/ssh.rs +++ b/crates/openshell-supervisor-process/src/ssh.rs @@ -6,7 +6,7 @@ use crate::child_env; #[cfg(target_os = "linux")] use crate::managed_children; -use crate::process::{drop_privileges, is_supervisor_only_env_var}; +use crate::process::{ProcessEnforcementMode, drop_privileges, is_supervisor_only_env_var}; use crate::sandbox; use miette::{IntoDiagnostic, Result}; use nix::pty::{Winsize, openpty}; @@ -42,6 +42,8 @@ type SshServerInit = ( fn ssh_server_init( listen_path: &Path, ca_file_paths: &Option<(PathBuf, PathBuf)>, + enforcement_mode: ProcessEnforcementMode, + shared_socket: bool, ) -> Result { let mut rng = OsRng; let host_key = PrivateKey::random(&mut rng, Algorithm::Ed25519).into_diagnostic()?; @@ -55,13 +57,17 @@ fn ssh_server_init( let config = Arc::new(config); let ca_paths = ca_file_paths.as_ref().map(|p| Arc::new(p.clone())); - // Ensure the parent directory exists and is root-owned with 0700 - // permissions. The sandbox entrypoint runs as an unprivileged user; it - // must not be able to enter this directory and connect to the socket. - if let Some(parent) = listen_path.parent() { + // In full enforcement mode the supervisor normally starts as root and can + // isolate the SSH socket in a root-only directory before spawning + // unprivileged children. Sidecar topology is different: the gateway relay + // runs in the network sidecar as a different UID, so the shared sidecar + // state directory must stay group-accessible. Sidecar mode uses a Linux + // abstract socket instead, so the workload cannot unlink the relay target. + let abstract_socket = crate::unix_socket::is_abstract(listen_path); + if !abstract_socket && let Some(parent) = listen_path.parent() { std::fs::create_dir_all(parent).into_diagnostic()?; #[cfg(unix)] - { + if enforcement_mode.uses_privileged_process_setup() && !shared_socket { use std::os::unix::fs::PermissionsExt; let perms = std::fs::Permissions::from_mode(0o700); std::fs::set_permissions(parent, perms).into_diagnostic()?; @@ -69,19 +75,19 @@ fn ssh_server_init( } // Remove any stale socket from a previous run before binding. - if listen_path.exists() { + if !abstract_socket && listen_path.exists() { std::fs::remove_file(listen_path).into_diagnostic()?; } - let listener = UnixListener::bind(listen_path).into_diagnostic()?; + let runtime_path = crate::unix_socket::runtime_path(listen_path); + let listener = UnixListener::bind(runtime_path.as_ref()).into_diagnostic()?; - // Tighten permissions so only the supervisor (root) can connect. The - // sandbox entrypoint runs as an unprivileged user and must not be able to - // dial the SSH daemon directly — all access goes through the relay from - // the gateway. + // Tighten filesystem-socket permissions. Abstract sockets have no inode; + // sidecar relay connections authenticate the listener with SO_PEERCRED. #[cfg(unix)] - { + if !abstract_socket { use std::os::unix::fs::PermissionsExt; - let perms = std::fs::Permissions::from_mode(0o600); + let mode = if shared_socket { 0o660 } else { 0o600 }; + let perms = std::fs::Permissions::from_mode(mode); std::fs::set_permissions(listen_path, perms).into_diagnostic()?; } @@ -108,8 +114,15 @@ pub async fn run_ssh_server( ca_file_paths: Option<(PathBuf, PathBuf)>, provider_credentials: ProviderCredentialState, user_environment: HashMap, + enforcement_mode: ProcessEnforcementMode, + shared_socket: bool, ) -> Result<()> { - let (listener, config, ca_paths) = match ssh_server_init(&listen_path, &ca_file_paths) { + let (listener, config, ca_paths) = match ssh_server_init( + &listen_path, + &ca_file_paths, + enforcement_mode, + shared_socket, + ) { Ok(v) => { // Signal that the SSH server has bound the socket and is ready to // accept connections. The parent task awaits this before spawning @@ -145,6 +158,7 @@ pub async fn run_ssh_server( ca_paths, provider_credentials, user_environment, + enforcement_mode, ) .await { @@ -172,6 +186,7 @@ async fn handle_connection( ca_file_paths: Option>, provider_credentials: ProviderCredentialState, user_environment: HashMap, + enforcement_mode: ProcessEnforcementMode, ) -> Result<()> { // Access is gated by the Unix-socket filesystem permissions (root-only), // not by an application-level preface. The supervisor bridges the @@ -195,6 +210,7 @@ async fn handle_connection( ca_file_paths, provider_credentials, user_environment, + enforcement_mode, ); russh::server::run_stream(config, stream, handler) .await @@ -223,6 +239,7 @@ struct SshHandler { ca_file_paths: Option>, provider_credentials: ProviderCredentialState, user_environment: HashMap, + enforcement_mode: ProcessEnforcementMode, channels: HashMap, } @@ -236,6 +253,7 @@ impl SshHandler { ca_file_paths: Option>, provider_credentials: ProviderCredentialState, user_environment: HashMap, + enforcement_mode: ProcessEnforcementMode, ) -> Self { Self { policy, @@ -245,6 +263,7 @@ impl SshHandler { ca_file_paths, provider_credentials, user_environment, + enforcement_mode, channels: HashMap::new(), } } @@ -468,6 +487,7 @@ impl russh::server::Handler for SshHandler { self.ca_file_paths.clone(), &self.provider_credentials.child_env_with_gcp_resolved(), &self.user_environment, + self.enforcement_mode, )?; let state = self.channels.get_mut(&channel).ok_or_else(|| { anyhow::anyhow!("subsystem_request on unknown channel {channel:?}") @@ -564,6 +584,7 @@ impl SshHandler { self.ca_file_paths.clone(), &provider_env, &self.user_environment, + self.enforcement_mode, )?; state.pty_master = Some(pty_master); state.input_sender = Some(input_sender); @@ -582,6 +603,7 @@ impl SshHandler { self.ca_file_paths.clone(), &provider_env, &self.user_environment, + self.enforcement_mode, )?; state.input_sender = Some(input_sender); } @@ -748,6 +770,7 @@ fn spawn_pty_shell( ca_file_paths: Option>, provider_env: &HashMap, user_environment: &HashMap, + enforcement_mode: ProcessEnforcementMode, ) -> anyhow::Result<(std::fs::File, mpsc::Sender>)> { let winsize = Winsize { ws_row: to_u16(pty.row_height.max(1)), @@ -806,12 +829,15 @@ fn spawn_pty_shell( // Probe Landlock availability from the parent process where tracing works. #[cfg(target_os = "linux")] - sandbox::linux::log_sandbox_readiness(policy, workdir.as_deref()); + if enforcement_mode.enforces_child_sandbox() { + sandbox::linux::log_sandbox_readiness(policy, workdir.as_deref()); + } - // Phase 1 (as root): Prepare Landlock ruleset before drop_privileges. + // Phase 1: Prepare Landlock ruleset before the child applies it. #[cfg(target_os = "linux")] - let prepared_sandbox = sandbox::linux::prepare(policy, workdir.as_deref()) - .map_err(|err| anyhow::anyhow!("Failed to prepare sandbox: {err}"))?; + let prepared_sandbox = + crate::process::prepare_child_sandbox(policy, workdir.as_deref(), enforcement_mode) + .map_err(|err| anyhow::anyhow!("Failed to prepare sandbox: {err}"))?; #[cfg(unix)] { @@ -821,6 +847,7 @@ fn spawn_pty_shell( workdir.clone(), slave_fd, netns_fd, + enforcement_mode, #[cfg(target_os = "linux")] prepared_sandbox, )?; @@ -913,6 +940,7 @@ fn spawn_pipe_exec( ca_file_paths: Option>, provider_env: &HashMap, user_environment: &HashMap, + enforcement_mode: ProcessEnforcementMode, ) -> anyhow::Result>> { let mut cmd = command.map_or_else( || { @@ -955,12 +983,15 @@ fn spawn_pipe_exec( // Probe Landlock availability from the parent process where tracing works. #[cfg(target_os = "linux")] - sandbox::linux::log_sandbox_readiness(policy, workdir.as_deref()); + if enforcement_mode.enforces_child_sandbox() { + sandbox::linux::log_sandbox_readiness(policy, workdir.as_deref()); + } - // Phase 1 (as root): Prepare Landlock ruleset before drop_privileges. + // Phase 1: Prepare Landlock ruleset before the child applies it. #[cfg(target_os = "linux")] - let prepared_sandbox = sandbox::linux::prepare(policy, workdir.as_deref()) - .map_err(|err| anyhow::anyhow!("Failed to prepare sandbox: {err}"))?; + let prepared_sandbox = + crate::process::prepare_child_sandbox(policy, workdir.as_deref(), enforcement_mode) + .map_err(|err| anyhow::anyhow!("Failed to prepare sandbox: {err}"))?; #[cfg(unix)] { @@ -969,6 +1000,7 @@ fn spawn_pipe_exec( policy.clone(), workdir.clone(), netns_fd, + enforcement_mode, #[cfg(target_os = "linux")] prepared_sandbox, )?; @@ -1068,7 +1100,9 @@ fn spawn_pipe_exec( mod unsafe_pty { #[cfg(not(target_os = "linux"))] use super::sandbox; - use super::{Command, RawFd, SandboxPolicy, Winsize, drop_privileges, setsid}; + use super::{ + Command, ProcessEnforcementMode, RawFd, SandboxPolicy, Winsize, drop_privileges, setsid, + }; #[cfg(unix)] use std::os::unix::process::CommandExt; @@ -1107,17 +1141,21 @@ mod unsafe_pty { _workdir: Option, slave_fd: RawFd, netns_fd: Option, - #[cfg(target_os = "linux")] prepared: crate::sandbox::linux::PreparedSandbox, + enforcement_mode: ProcessEnforcementMode, + #[cfg(target_os = "linux")] prepared: Option, ) -> anyhow::Result<()> { // Wrap in Option so we can .take() it out of the FnMut closure. // pre_exec is only called once (after fork, before exec). #[cfg(target_os = "linux")] - let mut prepared = Some(prepared); + let mut prepared = prepared; #[cfg(target_os = "linux")] - let supervisor_identity_mount = crate::process::supervisor_identity_mount_from_env() - .map_err(|err| { + let supervisor_identity_mount = if enforcement_mode.uses_privileged_process_setup() { + crate::process::supervisor_identity_mount_from_env().map_err(|err| { anyhow::anyhow!("failed to prepare supervisor identity isolation: {err}") - })?; + })? + } else { + None + }; unsafe { cmd.pre_exec(move || { setsid().map_err(|err| std::io::Error::other(err.to_string()))?; @@ -1126,6 +1164,7 @@ mod unsafe_pty { enter_netns_and_sandbox( netns_fd, &policy, + enforcement_mode, #[cfg(target_os = "linux")] supervisor_identity_mount, #[cfg(target_os = "linux")] @@ -1152,20 +1191,25 @@ mod unsafe_pty { policy: SandboxPolicy, _workdir: Option, netns_fd: Option, - #[cfg(target_os = "linux")] prepared: crate::sandbox::linux::PreparedSandbox, + enforcement_mode: ProcessEnforcementMode, + #[cfg(target_os = "linux")] prepared: Option, ) -> anyhow::Result<()> { #[cfg(target_os = "linux")] - let mut prepared = Some(prepared); + let mut prepared = prepared; #[cfg(target_os = "linux")] - let supervisor_identity_mount = crate::process::supervisor_identity_mount_from_env() - .map_err(|err| { + let supervisor_identity_mount = if enforcement_mode.uses_privileged_process_setup() { + crate::process::supervisor_identity_mount_from_env().map_err(|err| { anyhow::anyhow!("failed to prepare supervisor identity isolation: {err}") - })?; + })? + } else { + None + }; unsafe { cmd.pre_exec(move || { enter_netns_and_sandbox( netns_fd, &policy, + enforcement_mode, #[cfg(target_os = "linux")] supervisor_identity_mount, #[cfg(target_os = "linux")] @@ -1179,6 +1223,7 @@ mod unsafe_pty { fn enter_netns_and_sandbox( netns_fd: Option, policy: &SandboxPolicy, + enforcement_mode: ProcessEnforcementMode, #[cfg(target_os = "linux")] supervisor_identity_mount: Option< &crate::process::SupervisorIdentityMountNamespace, >, @@ -1207,7 +1252,9 @@ mod unsafe_pty { // Drop privileges. initgroups/setgid/setuid need /etc/group and // /etc/passwd which would be blocked if Landlock were already enforced. - drop_privileges(policy).map_err(|err| std::io::Error::other(err.to_string()))?; + if enforcement_mode.uses_privileged_process_setup() { + drop_privileges(policy).map_err(|err| std::io::Error::other(err.to_string()))?; + } crate::process::harden_child_process() .map_err(|err| std::io::Error::other(err.to_string()))?; @@ -1220,7 +1267,9 @@ mod unsafe_pty { } #[cfg(not(target_os = "linux"))] - sandbox::apply(policy, None).map_err(|err| std::io::Error::other(err.to_string()))?; + if enforcement_mode.enforces_child_sandbox() { + sandbox::apply(policy, None).map_err(|err| std::io::Error::other(err.to_string()))?; + } Ok(()) } @@ -1275,6 +1324,71 @@ mod tests { use super::*; use std::process::Stdio; + #[cfg(unix)] + fn file_mode(path: &Path) -> u32 { + use std::os::unix::fs::PermissionsExt; + std::fs::metadata(path).unwrap().permissions().mode() & 0o7777 + } + + #[cfg(unix)] + fn set_file_mode(path: &Path, mode: u32) { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode)).unwrap(); + } + + #[cfg(unix)] + #[tokio::test] + async fn ssh_server_init_full_enforcement_keeps_private_socket() { + let temp = tempfile::tempdir().unwrap(); + let parent = temp.path().join("ssh"); + std::fs::create_dir_all(&parent).unwrap(); + set_file_mode(&parent, 0o775); + let socket = parent.join("ssh.sock"); + + let (listener, _, _) = + ssh_server_init(&socket, &None, ProcessEnforcementMode::Full, false).unwrap(); + drop(listener); + + assert_eq!(file_mode(&parent), 0o700); + assert_eq!(file_mode(&socket), 0o600); + } + + #[cfg(unix)] + #[tokio::test] + async fn ssh_server_init_shared_socket_keeps_group_access() { + let temp = tempfile::tempdir().unwrap(); + let parent = temp.path().join("ssh"); + std::fs::create_dir_all(&parent).unwrap(); + set_file_mode(&parent, 0o775); + let socket = parent.join("ssh.sock"); + + let (listener, _, _) = + ssh_server_init(&socket, &None, ProcessEnforcementMode::Full, true).unwrap(); + drop(listener); + + assert_eq!(file_mode(&parent), 0o775); + assert_eq!(file_mode(&socket), 0o660); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn ssh_server_abstract_socket_cannot_be_replaced_while_bound() { + let socket = PathBuf::from(format!("@openshell-ssh-test-{}", uuid::Uuid::new_v4())); + let (listener, _, _) = + ssh_server_init(&socket, &None, ProcessEnforcementMode::NetworkOnly, true).unwrap(); + + assert!( + !socket.exists(), + "abstract socket must not create a filesystem inode" + ); + let runtime_path = crate::unix_socket::runtime_path(&socket); + let err = UnixListener::bind(runtime_path.as_ref()) + .expect_err("a workload must not be able to replace the bound abstract socket"); + assert_eq!(err.kind(), std::io::ErrorKind::AddrInUse); + + drop(listener); + } + /// Verify that dropping the input sender (the operation `channel_eof` /// performs) causes the stdin writer loop to exit and close the child's /// stdin pipe. Without this, commands like `cat | tar xf -` used by @@ -1681,21 +1795,24 @@ mod tests { policy, None, None, // no netns fd + ProcessEnforcementMode::Full, #[cfg(target_os = "linux")] - sandbox::linux::prepare( - &SandboxPolicy { - version: 0, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy::default(), - landlock: LandlockPolicy::default(), - process: ProcessPolicy { - run_as_user: None, - run_as_group: None, + Some( + sandbox::linux::prepare( + &SandboxPolicy { + version: 0, + filesystem: FilesystemPolicy::default(), + network: NetworkPolicy::default(), + landlock: LandlockPolicy::default(), + process: ProcessPolicy { + run_as_user: None, + run_as_group: None, + }, }, - }, - None, - ) - .expect("prepare should succeed in test environment"), + None, + ) + .expect("prepare should succeed in test environment"), + ), ) .expect("install pre_exec should succeed"); diff --git a/crates/openshell-supervisor-process/src/supervisor_session.rs b/crates/openshell-supervisor-process/src/supervisor_session.rs index 7c524676c6..594d865599 100644 --- a/crates/openshell-supervisor-process/src/supervisor_session.rs +++ b/crates/openshell-supervisor-process/src/supervisor_session.rs @@ -235,12 +235,14 @@ pub fn spawn( sandbox_id: String, ssh_socket_path: std::path::PathBuf, netns_fd: Option, + expected_ssh_peer_pid: Option, ) -> tokio::task::JoinHandle<()> { tokio::spawn(run_session_loop( endpoint, sandbox_id, ssh_socket_path, netns_fd, + expected_ssh_peer_pid, )) } @@ -249,6 +251,7 @@ async fn run_session_loop( sandbox_id: String, ssh_socket_path: std::path::PathBuf, netns_fd: Option, + expected_ssh_peer_pid: Option, ) { let mut backoff = INITIAL_BACKOFF; let mut attempt: u64 = 0; @@ -256,7 +259,15 @@ async fn run_session_loop( loop { attempt += 1; - match run_single_session(&endpoint, &sandbox_id, &ssh_socket_path, netns_fd).await { + match run_single_session( + &endpoint, + &sandbox_id, + &ssh_socket_path, + netns_fd, + expected_ssh_peer_pid, + ) + .await + { Ok(()) => { let event = session_closed_event(openshell_ocsf::ctx::ctx(), &endpoint, &sandbox_id); @@ -283,6 +294,7 @@ async fn run_single_session( sandbox_id: &str, ssh_socket_path: &std::path::Path, netns_fd: Option, + expected_ssh_peer_pid: Option, ) -> Result<(), Box> { // Connect to the gateway. The same `Channel` is used for both the // long-lived control stream and all data-plane `RelayStream` calls, so @@ -352,6 +364,7 @@ async fn run_single_session( sandbox_id, ssh_socket_path, netns_fd, + expected_ssh_peer_pid, &channel, &tx, ); @@ -375,6 +388,7 @@ fn handle_gateway_message( sandbox_id: &str, ssh_socket_path: &std::path::Path, netns_fd: Option, + expected_ssh_peer_pid: Option, channel: &grpc_client::AuthedChannel, tx: &mpsc::Sender, ) { @@ -395,7 +409,16 @@ fn handle_gateway_message( tokio::spawn(async move { let event_open = relay_open.clone(); - match handle_relay_open(relay_open, &ssh_socket_path, netns_fd, channel, tx).await { + match handle_relay_open( + relay_open, + &ssh_socket_path, + netns_fd, + expected_ssh_peer_pid, + channel, + tx, + ) + .await + { Ok(()) => { let event = relay_closed_event( openshell_ocsf::ctx::ctx(), @@ -446,11 +469,19 @@ async fn handle_relay_open( relay_open: RelayOpen, ssh_socket_path: &std::path::Path, netns_fd: Option, + expected_ssh_peer_pid: Option, channel: grpc_client::AuthedChannel, tx: mpsc::Sender, ) -> Result<(), Box> { let channel_id = relay_open.channel_id.clone(); - let target = match open_target(&relay_open, ssh_socket_path, netns_fd).await { + let target = match open_target( + &relay_open, + ssh_socket_path, + netns_fd, + expected_ssh_peer_pid, + ) + .await + { Ok(target) => target, Err(err) => { send_relay_open_result(&tx, &channel_id, false, err.to_string()).await; @@ -576,11 +607,23 @@ async fn open_target( relay_open: &RelayOpen, ssh_socket_path: &std::path::Path, netns_fd: Option, + expected_ssh_peer_pid: Option, ) -> Result, Box> { match relay_open.target.as_ref() { Some(relay_open::Target::Tcp(target)) => open_tcp_target(target, netns_fd).await, Some(relay_open::Target::Ssh(_)) | None => { - let stream = tokio::net::UnixStream::connect(ssh_socket_path).await?; + let runtime_path = crate::unix_socket::runtime_path(ssh_socket_path); + let stream = tokio::net::UnixStream::connect(runtime_path.as_ref()).await?; + if let Some(expected_pid) = expected_ssh_peer_pid { + let credentials = stream.peer_cred()?; + let actual_pid = credentials.pid().and_then(|pid| u32::try_from(pid).ok()); + if actual_pid != Some(expected_pid) { + return Err(format!( + "SSH relay peer PID mismatch: expected {expected_pid}, got {actual_pid:?}" + ) + .into()); + } + } Ok(Box::new(stream)) } } @@ -895,4 +938,37 @@ mod ocsf_event_tests { .expect_err("eof should force reconnect"); assert_eq!(err.to_string(), "gateway closed stream"); } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn ssh_target_requires_authenticated_supervisor_peer_pid() { + let socket = + std::path::PathBuf::from(format!("@openshell-relay-test-{}", uuid::Uuid::new_v4())); + let runtime_path = crate::unix_socket::runtime_path(&socket); + let listener = tokio::net::UnixListener::bind(runtime_path.as_ref()).unwrap(); + let accept_task = tokio::spawn(async move { + for _ in 0..2 { + let (_stream, _) = listener.accept().await.unwrap(); + } + }); + let relay = ssh_relay_open("peer-check"); + + let trusted = open_target(&relay, &socket, None, Some(std::process::id())) + .await + .expect("matching peer PID should be accepted"); + drop(trusted); + + let Err(err) = open_target( + &relay, + &socket, + None, + Some(std::process::id().saturating_add(1)), + ) + .await + else { + panic!("mismatched peer PID must be rejected"); + }; + assert!(err.to_string().contains("peer PID mismatch")); + accept_task.await.unwrap(); + } } diff --git a/crates/openshell-supervisor-process/src/unix_socket.rs b/crates/openshell-supervisor-process/src/unix_socket.rs new file mode 100644 index 0000000000..ba74e0a6bb --- /dev/null +++ b/crates/openshell-supervisor-process/src/unix_socket.rs @@ -0,0 +1,60 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Unix-socket address helpers shared by the SSH server and relay client. + +use std::borrow::Cow; +use std::path::Path; +#[cfg(target_os = "linux")] +use std::path::PathBuf; + +/// Return whether a configured socket name denotes a Linux abstract socket. +/// +/// Environment variables cannot contain the leading NUL byte used by the +/// kernel ABI, so configuration uses the conventional `@name` spelling. +pub fn is_abstract(path: &Path) -> bool { + #[cfg(target_os = "linux")] + { + use std::os::unix::ffi::OsStrExt; + path.as_os_str().as_bytes().starts_with(b"@") + } + #[cfg(not(target_os = "linux"))] + { + let _ = path; + false + } +} + +/// Translate the configured `@name` spelling into Tokio's NUL-prefixed Linux +/// abstract-socket path representation. +pub fn runtime_path(path: &Path) -> Cow<'_, Path> { + #[cfg(target_os = "linux")] + { + use std::ffi::OsString; + use std::os::unix::ffi::{OsStrExt, OsStringExt}; + + let bytes = path.as_os_str().as_bytes(); + if let Some(name) = bytes.strip_prefix(b"@") { + let mut abstract_name = Vec::with_capacity(name.len() + 1); + abstract_name.push(0); + abstract_name.extend_from_slice(name); + return Cow::Owned(PathBuf::from(OsString::from_vec(abstract_name))); + } + } + Cow::Borrowed(path) +} + +#[cfg(all(test, target_os = "linux"))] +mod tests { + use super::*; + + #[test] + fn at_name_maps_to_abstract_socket_path() { + use std::os::unix::ffi::OsStrExt; + + let configured = Path::new("@openshell-test"); + let runtime = runtime_path(configured); + assert!(is_abstract(configured)); + assert_eq!(runtime.as_os_str().as_bytes(), b"\0openshell-test"); + } +} diff --git a/deploy/docker/Dockerfile.supervisor b/deploy/docker/Dockerfile.supervisor index c84cc70e9e..c760bbc890 100644 --- a/deploy/docker/Dockerfile.supervisor +++ b/deploy/docker/Dockerfile.supervisor @@ -5,10 +5,10 @@ # Supervisor image build. # -# The final image is `scratch`: it only carries the static `openshell-sandbox` -# binary used by Docker extraction, Podman image volumes, and the Kubernetes -# init container copy-self path. A static musl binary lets the image stay -# `scratch` while still being executable as an init container. +# The final image carries the static `openshell-sandbox` binary used by Docker +# extraction, Podman image volumes, and the Kubernetes init container copy-self +# path. It also includes nftables so the Kubernetes supervisor sidecar can +# install pod-namespace egress enforcement rules. # # The Rust binary is built natively before this image build runs and staged at: # deploy/docker/.build/prebuilt-binaries//openshell-sandbox @@ -19,17 +19,16 @@ # target) and uploads it as an artifact, which is downloaded into the same # staging directory before the image build job runs. -FROM scratch AS supervisor +FROM alpine:3.22 AS supervisor ARG TARGETARCH -# --chmod=0550 drops world-execute and survives the actions/upload-artifact -# + download-artifact roundtrip (which strips exec perms). Ownership is left -# at root (0:0) deliberately: the Podman driver mounts this image as a -# read-only image volume into the sandbox container and drops DAC_OVERRIDE, -# so the container's UID 0 must own the binary to read+exec it. Mode 0550 -# (r-xr-x---) is the security win; the chown to a non-root UID was breaking -# Podman without buying anything since the container is always UID 0. -COPY --chmod=0550 deploy/docker/.build/prebuilt-binaries/${TARGETARCH}/openshell-sandbox /openshell-sandbox +RUN apk add --no-cache nftables iptables iptables-legacy + +# --chmod=0555 restores execute bits after the actions/upload-artifact + +# download-artifact roundtrip strips them. Ownership stays root (0:0) for +# Podman image-volume mounts, while world-execute lets the Kubernetes +# network sidecar run this binary as the dedicated non-root proxy UID. +COPY --chmod=0555 deploy/docker/.build/prebuilt-binaries/${TARGETARCH}/openshell-sandbox /openshell-sandbox ENTRYPOINT ["/openshell-sandbox"] diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index 4a22d640f5..8723535d1a 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -239,8 +239,10 @@ add `ci/values-spire.yaml` to the OpenShell release values files. | supervisor.image.pullPolicy | string | `""` | Supervisor image pull policy. Defaults to the gateway image pull policy when empty. | | supervisor.image.repository | string | `"ghcr.io/nvidia/openshell/supervisor"` | Supervisor image repository. Changing it uses the effective gateway image tag unless tag is also set. | | supervisor.image.tag | string | `""` | Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed. | +| supervisor.sidecar.processBinaryAwareNetworkPolicy | bool | `true` | Keep process/binary-aware network policy enabled in sidecar topology. When false, the network sidecar runs as proxyUid, drops the extra /proc inspection capabilities, and enforces endpoint/L7 policy without matching policy.binaries. | +| supervisor.sidecar.proxyUid | int | `1337` | UID for relaxed long-running network sidecars in sidecar topology. Strict process/binary-aware sidecars run as UID 0 so Kubernetes grants the required /proc inspection capabilities into the effective set. The network init container installs nftables rules that exempt the effective sidecar UID. | | supervisor.sideloadMethod | string | `""` | How the supervisor binary is delivered into sandbox pods. Empty (default) = auto-detect from cluster version: K8s >= v1.35 -> "image-volume" (ImageVolume enabled by default; GA in v1.36) K8s < v1.35 -> "init-container" (copies via init container + emptyDir) On K8s v1.33-v1.34 with the ImageVolume feature gate manually enabled, set this to "image-volume" explicitly. | -| supervisor.topology | string | `"combined"` | Supervisor pod topology for Kubernetes sandboxes. "combined" runs networking and process supervision in the agent container. | +| supervisor.topology | string | `"combined"` | Supervisor pod topology for Kubernetes sandboxes. "combined" runs the current single supervisor container in the agent pod. "sidecar" runs network enforcement in a dedicated sidecar and the process supervisor as a low-capability wrapper in the agent container. | | tolerations | list | `[]` | Tolerations for the gateway pod. | | workload.allowMultiReplicaStatefulSet | bool | `false` | Allow replicaCount > 1 while rendering a StatefulSet. Prefer workload.kind=deployment for external database-backed multi-replica gateways; this override exists for operators who explicitly require StatefulSet identity or storage semantics. | | workload.kind | string | `"statefulset"` | Gateway workload controller kind. Use `statefulset` for the default SQLite database, or `deployment` when server.externalDbSecret points at an external database. | diff --git a/deploy/helm/openshell/ci/values-sidecar-kata.yaml b/deploy/helm/openshell/ci/values-sidecar-kata.yaml new file mode 100644 index 0000000000..2e23a1009e --- /dev/null +++ b/deploy/helm/openshell/ci/values-sidecar-kata.yaml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# CI/dev overlay for exercising the Kubernetes supervisor sidecar topology under +# a Kata RuntimeClass. +# +# Use with e2e/with-kube-gateway.sh by setting: +# OPENSHELL_E2E_KUBE_EXTRA_VALUES=deploy/helm/openshell/ci/values-sidecar-kata.yaml +# The e2e wrapper supplies the image repository and tag through OPENSHELL_REGISTRY +# and IMAGE_TAG for existing-cluster runs. + +supervisor: + # Use the sidecar topology under Kata so network enforcement runs in the + # sidecar and the sandbox agent container stays low-privilege. + topology: sidecar + sidecar: + # Keep strict process/binary-aware network policy enabled for the Kata + # validation path. Set this false only when intentionally validating the + # documented endpoint/L7-only downgrade mode. + processBinaryAwareNetworkPolicy: true + +# Kata validation clusters normally install this RuntimeClass. +server: + defaultRuntimeClassName: kata-qemu diff --git a/deploy/helm/openshell/ci/values-sidecar.yaml b/deploy/helm/openshell/ci/values-sidecar.yaml new file mode 100644 index 0000000000..ba8dc50ef1 --- /dev/null +++ b/deploy/helm/openshell/ci/values-sidecar.yaml @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# CI/dev overlay for exercising the Kubernetes supervisor sidecar topology. +# +# Merge after values.yaml and ci/values-skaffold.yaml: +# helm install ... -f values.yaml -f ci/values-skaffold.yaml -f ci/values-sidecar.yaml +# +# Or set: +# OPENSHELL_E2E_KUBE_EXTRA_VALUES=deploy/helm/openshell/ci/values-sidecar.yaml +# before running `mise run e2e:kubernetes`. +supervisor: + topology: sidecar + sidecar: + # The strict sidecar default requires cross-container /proc identity access. + # CI/dev e2e uses the explicit downgraded mode so endpoint and L7 policy + # coverage remains runnable on local k3d while that path is hardened. + processBinaryAwareNetworkPolicy: false diff --git a/deploy/helm/openshell/skaffold.yaml b/deploy/helm/openshell/skaffold.yaml index d571c3bd9e..119adf086b 100644 --- a/deploy/helm/openshell/skaffold.yaml +++ b/deploy/helm/openshell/skaffold.yaml @@ -119,6 +119,8 @@ deploy: # To enable SPIFFE/SPIRE provider token grants (requires the # spire-crds and spire releases above): #- ci/values-spire.yaml + # To exercise the Kubernetes supervisor sidecar topology: + #- ci/values-sidecar.yaml # To test multi-replica external PostgreSQL behavior: #- ci/values-high-availability.yaml setValueTemplates: @@ -126,3 +128,18 @@ deploy: image.tag: '{{.IMAGE_TAG_openshell_gateway}}' supervisor.image.repository: '{{.IMAGE_REPO_openshell_supervisor}}' supervisor.image.tag: '{{.IMAGE_TAG_openshell_supervisor}}' +profiles: + - name: sidecar + patches: + - op: add + path: /deploy/helm/releases/0/valuesFiles/- + value: ci/values-sidecar.yaml + - name: sidecar-mtls + patches: + - op: add + path: /deploy/helm/releases/0/valuesFiles/- + value: ci/values-sidecar.yaml + - op: add + path: /deploy/helm/releases/0/setValues + value: + server.disableTls: "false" diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 55beac7a19..36a579250b 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -115,7 +115,7 @@ data: grpc_endpoint = {{ include "openshell.grpcEndpoint" . | quote }} service_account_name = {{ include "openshell.sandboxServiceAccountName" . | quote }} supervisor_sideload_method = {{ include "openshell.supervisorSideloadMethod" . | quote }} - supervisor_topology = {{ .Values.supervisor.topology | default "combined" | quote }} + topology = {{ .Values.supervisor.topology | default "combined" | quote }} sa_token_ttl_secs = {{ .Values.server.sandboxJwt.k8sSaTokenTtlSecs | default 3600 }} {{- if .Values.server.providerTokenGrants.spiffe.enabled }} provider_spiffe_workload_api_socket_path = {{ .Values.server.providerTokenGrants.spiffe.workloadApiSocketPath | quote }} @@ -144,3 +144,7 @@ data: {{- if .Values.supervisor.image.pullPolicy }} supervisor_image_pull_policy = {{ .Values.supervisor.image.pullPolicy | quote }} {{- end }} + + [openshell.drivers.kubernetes.sidecar] + proxy_uid = {{ .Values.supervisor.sidecar.proxyUid | default 1337 }} + process_binary_aware_network_policy = {{ .Values.supervisor.sidecar.processBinaryAwareNetworkPolicy }} diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index 797182558e..aee396c38f 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -88,7 +88,10 @@ tests: asserts: - matchRegex: path: data["gateway.toml"] - pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?supervisor_topology\s*=\s*"combined"' + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?topology\s*=\s*"combined"' + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'supervisor[_]topology\s*=' - it: uses the gateway built-in supervisor image by default template: templates/gateway-config.yaml @@ -126,14 +129,35 @@ tests: path: data["gateway.toml"] pattern: 'supervisor_image\s*=\s*"registry\.example\.com/openshell/supervisor:supervisor-build"' - - it: renders explicit combined supervisor topology under [openshell.drivers.kubernetes] + - it: renders sidecar supervisor topology under [openshell.drivers.kubernetes] + template: templates/gateway-config.yaml + set: + supervisor.topology: sidecar + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?topology\s*=\s*"sidecar"' + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'supervisor[_]topology\s*=' + + - it: renders proxy uid under [openshell.drivers.kubernetes.sidecar] + template: templates/gateway-config.yaml + set: + supervisor.sidecar.proxyUid: 2200 + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\.sidecar\].*?proxy_uid\s*=\s*2200' + + - it: renders process binary aware network policy under [openshell.drivers.kubernetes.sidecar] template: templates/gateway-config.yaml set: - supervisor.topology: combined + supervisor.sidecar.processBinaryAwareNetworkPolicy: false asserts: - matchRegex: path: data["gateway.toml"] - pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?supervisor_topology\s*=\s*"combined"' + pattern: '(?ms)\[openshell\.drivers\.kubernetes\.sidecar\].*?process_binary_aware_network_policy\s*=\s*false' - it: renders sandbox image pull secrets under [openshell.drivers.kubernetes] template: templates/gateway-config.yaml diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index f36b533d13..e89a234912 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -45,8 +45,22 @@ supervisor: # set this to "image-volume" explicitly. sideloadMethod: "" # -- Supervisor pod topology for Kubernetes sandboxes. - # "combined" runs networking and process supervision in the agent container. + # "combined" runs the current single supervisor container in the agent pod. + # "sidecar" runs network enforcement in a dedicated sidecar and the process + # supervisor as a low-capability wrapper in the agent container. topology: "combined" + sidecar: + # -- UID for relaxed long-running network sidecars in sidecar topology. + # Strict process/binary-aware sidecars run as UID 0 so Kubernetes grants + # the required /proc inspection capabilities into the effective set. The + # network init container installs nftables rules that exempt the effective + # sidecar UID. + proxyUid: 1337 + # -- Keep process/binary-aware network policy enabled in sidecar topology. + # When false, the network sidecar runs as proxyUid, drops the extra /proc + # inspection capabilities, and enforces endpoint/L7 policy without matching + # policy.binaries. + processBinaryAwareNetworkPolicy: true # -- Image pull secrets attached to gateway and helper pods. imagePullSecrets: [] diff --git a/docs/kubernetes/access-control.mdx b/docs/kubernetes/access-control.mdx index 8824b6de1e..5409a4b11d 100644 --- a/docs/kubernetes/access-control.mdx +++ b/docs/kubernetes/access-control.mdx @@ -5,7 +5,7 @@ title: "Access Control" sidebar-title: "Access Control" description: "Configure OIDC user authentication or reverse-proxy auth termination for a Kubernetes-deployed OpenShell gateway." keywords: "Generative AI, Cybersecurity, Kubernetes, Authentication, mTLS, OIDC, Keycloak, Entra ID, Okta, Gateway Auth" -position: 4 +position: 5 --- The OpenShell gateway supports two access-control models for human callers on Kubernetes: diff --git a/docs/kubernetes/ingress.mdx b/docs/kubernetes/ingress.mdx index 844167246a..66178bc3a0 100644 --- a/docs/kubernetes/ingress.mdx +++ b/docs/kubernetes/ingress.mdx @@ -5,7 +5,7 @@ title: "Ingress" sidebar-title: "Ingress" description: "Expose the OpenShell gateway externally using the Kubernetes Gateway API and a GRPCRoute." keywords: "Generative AI, Cybersecurity, Kubernetes, Gateway API, Envoy Gateway, GRPCRoute, Ingress, External Access" -position: 3 +position: 4 --- By default, the OpenShell gateway is only reachable inside the cluster. To let CLI clients connect without a `kubectl port-forward`, expose the gateway through an ingress. diff --git a/docs/kubernetes/managing-certificates.mdx b/docs/kubernetes/managing-certificates.mdx index d929463504..b66419b505 100644 --- a/docs/kubernetes/managing-certificates.mdx +++ b/docs/kubernetes/managing-certificates.mdx @@ -5,7 +5,7 @@ title: "Managing Certificates" sidebar-title: "Managing Certificates" description: "Configure the OpenShell Helm chart to use cert-manager for mTLS certificate issuance and automatic renewal." keywords: "Generative AI, Cybersecurity, Kubernetes, cert-manager, PKI, TLS, mTLS, Certificates" -position: 2 +position: 3 --- The OpenShell gateway uses mTLS certificates for transport between the gateway and sandbox supervisors. These certificates are not Kubernetes user authentication; configure OIDC or a trusted access proxy for user access. The Helm chart supports two ways to provision and manage the certificate bundle: diff --git a/docs/kubernetes/openshift.mdx b/docs/kubernetes/openshift.mdx index 355a46d2f6..7512eaa65e 100644 --- a/docs/kubernetes/openshift.mdx +++ b/docs/kubernetes/openshift.mdx @@ -5,7 +5,7 @@ title: "OpenShift" sidebar-title: "OpenShift" description: "Install the OpenShell Helm chart on OpenShift, including the SCC binding and chart overrides required by OpenShift's Security Context Constraints." keywords: "Generative AI, Cybersecurity, Kubernetes, OpenShift, SCC, Security Context Constraints, Helm, Gateway, Installation" -position: 5 +position: 6 --- @@ -14,6 +14,11 @@ The OpenShift install path is experimental. It currently requires running sandbo OpenShift's [Security Context Constraints](https://docs.openshift.com/container-platform/latest/authentication/managing-security-context-constraints.html) reject the chart's default pod security settings. Installing on OpenShift requires precreating the namespace, granting the `privileged` SCC to the sandbox service account, and overriding a few chart values so the cluster admission controller can assign UIDs and FS groups itself. +OpenShell installs sandbox nftables rules as individual commands. On OpenShift +nodes where optional conntrack or packet log expressions are unavailable, those +optional rules can fail without rolling back the required proxy bypass reject +rules. + ## Prerequisites - OpenShift 4.x cluster with `oc` configured diff --git a/docs/kubernetes/setup.mdx b/docs/kubernetes/setup.mdx index f6051b1235..c2fca827f1 100644 --- a/docs/kubernetes/setup.mdx +++ b/docs/kubernetes/setup.mdx @@ -161,6 +161,7 @@ The most commonly changed values are: | `pkiInitJob.serverDnsNames` / `certManager.serverDnsNames` | Additional gateway server DNS SANs. Wildcard SANs also enable sandbox service URLs under that domain. | | `supervisor.sideloadMethod` | How the supervisor binary is delivered into sandbox pods. Leave empty to auto-detect based on cluster version: clusters running Kubernetes 1.35 or later use `image-volume` (ImageVolume GA in 1.36); older clusters use `init-container`. Set explicitly to `image-volume` on Kubernetes 1.33 or 1.34 with the ImageVolume feature gate enabled, or to `init-container` to force the legacy path on any version. | | `supervisor.topology` | Sandbox pod topology. Refer to [Topology](/kubernetes/topology). | +| `supervisor.sidecar.proxyUid` | Non-root UID used when sidecar process/binary-aware network policy is disabled. The default binary-aware sidecar runs as UID 0 instead. The configured UID must not match the sandbox UID. | Use a values file for repeatable deployments: @@ -244,7 +245,7 @@ The gateway exposes `/healthz` for process liveness and `/readyz` for dependency ## Next Steps -- To review Kubernetes sandbox topology, refer to [Topology](/kubernetes/topology). +- To choose between combined and sidecar sandbox pods, refer to [Topology](/kubernetes/topology). - To enable automatic certificate rotation with cert-manager, refer to [Managing Certificates](/kubernetes/managing-certificates). - To expose the gateway externally without port-forwarding, refer to [Ingress](/kubernetes/ingress). - To configure OIDC or reverse-proxy authentication, refer to [Access Control](/kubernetes/access-control). diff --git a/docs/kubernetes/topology.mdx b/docs/kubernetes/topology.mdx index f3f69cf6ec..5bbb18e1ef 100644 --- a/docs/kubernetes/topology.mdx +++ b/docs/kubernetes/topology.mdx @@ -3,37 +3,36 @@ # SPDX-License-Identifier: Apache-2.0 title: "Kubernetes Sandbox Topology" sidebar-title: "Topology" -description: "Review the default combined supervisor topology for Kubernetes sandbox pods." -keywords: "Generative AI, Cybersecurity, Kubernetes, Sandboxing, RuntimeClass" +description: "Choose between combined and sidecar supervisor topology for Kubernetes sandbox pods." +keywords: "Generative AI, Cybersecurity, Kubernetes, Sandboxing, Sidecar, Network Policy, RuntimeClass" position: 2 --- -Kubernetes sandbox pods run the OpenShell supervisor in `combined` topology by -default. Combined topology keeps network, filesystem, and process controls in -the agent pod so the supervisor can enforce the complete OpenShell sandbox -contract before launching the workload. +Kubernetes sandbox pods can run the OpenShell supervisor in `combined` or +`sidecar` topology. Choose the topology based on which controls you need inside +the pod and how much privilege your cluster allows on the agent container. ## Choose a Topology The default `combined` topology preserves the full OpenShell enforcement model. -Use it when you need OpenShell to apply all sandbox controls inside the workload -pod and your cluster policy permits the required Linux capabilities. +Use `sidecar` only when you accept network-focused enforcement in exchange for a +lower-privilege agent container. | Topology | Use when | Main tradeoff | |---|---|---| | `combined` | You need OpenShell network, filesystem, and process controls in the sandbox workload. | The agent container carries the Linux capabilities the supervisor needs. | - -Additional Kubernetes sandbox topologies are still being designed. Until they -are documented as supported configuration values, `combined` is the only -supported value for `supervisor.topology`. +| `sidecar` | You need the agent container to run as non-root without added Linux capabilities, and network policy is the primary control. | Privilege-dropping and supervisor mount isolation do not run in the agent container. | ## Privilege Model -The long-running container permissions for `combined` topology are: +The long-running container permissions differ by topology: | Topology | Pod or container | UID/GID | Privilege escalation | Capabilities | Result | |---|---|---|---|---|---| | `combined` | Agent container, which also runs the supervisor | Not forced by topology | Not explicitly disabled by the driver | Adds `SYS_ADMIN`, `NET_ADMIN`, `SYS_PTRACE`, and `SYSLOG`; adds `SETUID`, `SETGID`, and `DAC_READ_SEARCH` when user namespaces are enabled | Full supervisor controls run in the agent container. | +| `sidecar` | Agent container, process-only supervisor (`network-only`) | `sandbox_uid:sandbox_gid` | `false` | Drops `ALL` | Agent and workload run without added Linux capabilities. | +| `sidecar` | Network supervisor sidecar, binary-aware mode (default) | `0:sandbox_gid` | `false` | Drops `ALL`; adds `SYS_PTRACE` and `DAC_READ_SEARCH` | Root sidecar inspects cross-UID workload `/proc` entries. The nftables fence exempts UID 0, so do not inject other root containers into these pods. | +| `sidecar` | Network supervisor sidecar, endpoint/L7-only mode | `proxyUid:sandbox_gid` | `false` | Drops `ALL` | Non-root sidecar enforces endpoint and L7 policy without matching `policy.binaries`. | Short-lived setup containers still have the permissions needed to prepare the pod: @@ -41,6 +40,7 @@ pod: | Topology | Setup container | UID/GID | Privilege escalation | Capabilities | Purpose | |---|---|---|---|---|---| | `combined` | Supervisor install init container | `0` | Not set | Not set | Copies the supervisor binary into the agent container volume. | +| `sidecar` | Network init container | `0` | `false` | Drops `ALL`; adds `NET_ADMIN`, `NET_RAW`, `CHOWN`, and `FOWNER` | Installs pod-local nftables rules and prepares shared sidecar state. | ## Combined Topology @@ -48,6 +48,26 @@ Combined topology is the original Kubernetes mode and remains the default. The agent container starts the OpenShell supervisor, and the supervisor launches the workload after applying sandbox setup. +```mermaid +flowchart TB + Sandbox["agents.x-k8s.io Sandbox"] + + subgraph Pod["Sandbox pod"] + subgraph Agent["agent container"] + Supervisor["OpenShell supervisor
network + process + filesystem"] + Workload["Agent workload"] + end + end + + Gateway["OpenShell Gateway"] + External["External services"] + + Sandbox --> Pod + Supervisor --> Workload + Supervisor -->|"gateway callback / SSH relay"| Gateway + Supervisor -->|"policy-enforced egress"| External +``` + Combined topology keeps these controls in one supervisor path: - Network endpoint and L7 policy enforcement. @@ -61,12 +81,119 @@ controls from the agent container, Kubernetes grants that container elevated Linux capabilities. Use this mode when you need the complete OpenShell sandbox contract and your cluster policy permits those capabilities. +## Sidecar Topology + +Sidecar topology splits the supervisor into a network sidecar and a +low-privilege process supervisor in the agent container. + +```mermaid +flowchart TB + Sandbox["agents.x-k8s.io Sandbox"] + + subgraph Pod["Sandbox pod"] + Init["network init container
root setup capabilities"] + State["shared state + TLS volumes"] + NetNS["pod network namespace"] + + subgraph Agent["agent container"] + ProcessSupervisor["process supervisor
network-only"] + Workload["Agent workload"] + end + + NetworkSidecar["network supervisor sidecar
UID 0 by default"] + SshEndpoint["abstract SSH relay socket
peer-PID authenticated"] + end + + Gateway["OpenShell Gateway"] + External["External services"] + + Sandbox --> Pod + Init -->|"installs nftables rules"| NetNS + ProcessSupervisor --> Workload + Workload -->|"egress redirected on loopback"| NetworkSidecar + NetworkSidecar -->|"gateway session + relays"| Gateway + NetworkSidecar -->|"policy-enforced egress"| External + NetworkSidecar -->|"control socket + proxy TLS"| State + ProcessSupervisor -->|"bootstrap + updates"| State + ProcessSupervisor --> SshEndpoint + NetworkSidecar -->|"SSH relay"| SshEndpoint + NetworkSidecar --- State +``` + +The pod contains these OpenShell-managed pieces: + +| Component | Runs as | Purpose | +|---|---|---| +| Network init container | Root with setup capabilities | Installs pod-level nftables rules and prepares shared sidecar state. | +| Network sidecar | UID 0 by default; `supervisor.sidecar.proxyUid` when binary-aware policy is disabled | Runs the proxy, enforces network policy, owns gateway authentication and the gateway session, and serves local policy/provider state over the sidecar control socket. | +| Agent container | Resolved sandbox UID/GID | Runs the process supervisor and launches the user workload. | + +In this topology, the agent container defaults to `runAsNonRoot: true`, +`allowPrivilegeEscalation: false`, and `capabilities.drop: ["ALL"]`. The +default binary-aware network sidecar runs as UID 0, drops default Linux +capabilities, and adds `SYS_PTRACE` plus `DAC_READ_SEARCH` for cross-UID workload +process identity resolution. Setting +`supervisor.sidecar.processBinaryAwareNetworkPolicy=false` runs the sidecar as +the configured non-root `proxyUid`, omits both capabilities, and downgrades +network policy to endpoint/L7 enforcement without binary matching. The root +init container keeps the setup capabilities needed to configure pod networking. + +Sidecar mode preserves gateway session behavior, including SSH connectivity, +because the network sidecar owns the gateway session and bridges relay requests +to a Linux abstract SSH socket owned by the process supervisor. The relay +verifies the socket peer PID against the authenticated control connection, so +the workload cannot replace the relay endpoint. The agent container does not get a +gateway endpoint, gateway TLS material, or the sandbox bootstrap token in the +default sidecar path. + + +Sidecar mode runs the process supervisor in `network-only` mode. OpenShell still +enforces network endpoint and L7 policy through the sidecar, and the process +supervisor applies Landlock filesystem policy and child seccomp filters where +the kernel/runtime supports them. The process supervisor does not perform +root-to-sandbox privilege dropping because Kubernetes starts the container as +the sandbox UID/GID, and it does not perform supervisor identity mount +isolation because gateway credentials are not mounted into the agent container. +Sidecar pods use `shareProcessNamespace: true` so the network sidecar can +resolve workload process and binary identity through `/proc/`. + + +## Credential Exposure + +Sidecar topology keeps gateway credentials in the network sidecar. The agent +container does not mount the projected ServiceAccount token used for sandbox +token bootstrap, does not mount the sandbox client TLS secret, and does not get +gateway callback environment variables. + +The network sidecar serves the policy and workload-facing provider environment +over a Unix control socket in the shared sidecar state volume. Before launching +the workload, the process supervisor establishes the only accepted connection. +The sidecar validates its UID, GID, and PID with peer credentials, unlinks the +listener, derives the SSH target from trusted configuration, and rejects later +clients. The connection receives bootstrap state and provider-environment +updates after settings polls. If it closes, the network sidecar exits so +Kubernetes recreates the one-client bootstrap listener, and the process +supervisor exits so Kubernetes terminates the workload and restarts the agent +container. This symmetric failure behavior prevents a surviving workload from +claiming the new control listener after an isolated sidecar restart. Future +child processes can see refreshed provider env without giving the agent +container gateway authentication material. This does not mutate the environment +of the already-running workload entrypoint. Use `combined` topology when you +need the full single-supervisor enforcement path; use additional runtime +isolation when you need a stronger container boundary around sidecar workloads. + ## RuntimeClass Isolation -RuntimeClass isolation can add a stronger container boundary for the sandbox -workload when the cluster supports it. Runtime classes do not replace the -combined topology's supervisor controls; they add another isolation boundary -around the same supervised workload. +Sidecar topology has been validated with Kata Containers. It does not currently +support gVisor because sidecar mode requires pod-local nftables setup, which +gVisor does not provide to the init container. A supported sandboxed runtime +strengthens the container boundary while OpenShell focuses on network policy +enforcement from the sidecar. + +Runtime classes do not re-enable the OpenShell privilege-drop or supervisor +mount-isolation controls that sidecar mode relaxes. Use them as an additional +workload boundary, not as a replacement for the combined topology's full +supervisor controls. You can set a default runtime class in the Kubernetes driver configuration or override it per sandbox with driver config: @@ -77,24 +204,44 @@ openshell sandbox create \ -- claude ``` -## Configure Combined Mode +## Enable Sidecar Mode -For direct gateway TOML configuration, leave `supervisor_topology` unset, or -set it to `combined`, to use the default single-container supervisor path: +For direct gateway TOML configuration, set the Kubernetes driver fields: ```toml [openshell.drivers.kubernetes] -supervisor_topology = "combined" +topology = "sidecar" + +[openshell.drivers.kubernetes.sidecar] +proxy_uid = 1337 ``` -When the Helm chart renders `gateway.toml`, leave `supervisor.topology` unset, -or set it to `combined`, to produce the same driver configuration: +`proxy_uid` configures only the relaxed endpoint/L7-only sidecar. It must be a +non-root UID and must not match the sandbox UID. The default binary-aware mode +runs the sidecar as UID 0 instead. The network init container exempts the +effective sidecar UID from proxy redirection so the sidecar can reach the +gateway. + +When the Helm chart renders `gateway.toml`, set the equivalent chart values: ```yaml supervisor: - topology: combined + topology: sidecar + sidecar: + proxyUid: 1337 + processBinaryAwareNetworkPolicy: true ``` +Leave `topology` unset, or set it to `combined`, to keep the original +single-container supervisor path. For Helm installs, leave +`supervisor.topology` unset or set it to `combined`. + +Set `supervisor.sidecar.processBinaryAwareNetworkPolicy=false` only when you +accept downgrading sidecar network policy to endpoint/L7 enforcement without +matching `policy.binaries`. This changes the sidecar from UID 0 to `proxyUid` +and removes its `SYS_PTRACE` and `DAC_READ_SEARCH` capabilities, which are used +for cross-UID `/proc` inspection. + ## Next Steps - To install OpenShell on Kubernetes, refer to [Setup](/kubernetes/setup). diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 8f840d1776..247774a6c0 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -179,9 +179,10 @@ supervisor_image_pull_policy = "IfNotPresent" # Use the image volume on Kubernetes >= 1.35 (GA in 1.36); switch to "init-container" # on older clusters or where the ImageVolume feature gate is off. supervisor_sideload_method = "image-volume" -# "combined" runs networking and process supervision in the sandbox agent -# container and preserves the existing Kubernetes sandbox behavior. -supervisor_topology = "combined" +# "combined" runs the existing single supervisor container with full process, +# filesystem, and network enforcement in the agent container. "sidecar" moves +# pod-level network enforcement and gateway session handling into a network sidecar. +topology = "combined" grpc_endpoint = "https://openshell-gateway.agents.svc:8080" ssh_socket_path = "/run/openshell/ssh.sock" client_tls_secret_name = "openshell-client-tls" @@ -205,6 +206,18 @@ provider_spiffe_workload_api_socket_path = "/spiffe-workload-api/spire-agent.soc # back to 1000 on non-OpenShift clusters. # sandbox_uid = 1500 # sandbox_gid = 1500 + +[openshell.drivers.kubernetes.sidecar] +# UID used by relaxed long-running network sidecars. Strict process/binary-aware +# sidecars run as UID 0 so Kubernetes grants the required /proc inspection +# capabilities into the effective set. In sidecar topology the network init +# container installs nftables rules that exempt the effective sidecar UID. +proxy_uid = 1337 +# Keep process/binary-aware network policy enabled in sidecar topology. Set +# false to run the sidecar as proxy_uid, drop the sidecar's extra /proc +# inspection capabilities, and enforce endpoint/L7 policy without matching +# policy.binaries. +process_binary_aware_network_policy = true ``` ### Docker diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index e5cb8e254d..dc8bdb2345 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -306,10 +306,38 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA | `supervisor_image` | `supervisor.image.repository` / `supervisor.image.tag` | Override the supervisor image that provides the `openshell-sandbox` binary. The default repository with an empty tag uses the version-pinned image built into the gateway. Changing the repository uses the effective gateway image tag, while setting a tag pins that version explicitly. | | `supervisor_image_pull_policy` | `supervisor.image.pullPolicy` | Set the Kubernetes image pull policy for the supervisor image. | | `supervisor_sideload_method` | `supervisor.sideloadMethod` | How the supervisor binary is delivered into sandbox pods. Leave empty to auto-detect from cluster version. Set to `image-volume` to mount the supervisor OCI image directly as a volume (requires Kubernetes 1.33+ with the ImageVolume feature gate; GA in 1.36), or `init-container` to copy it through an init container on older clusters. | +| `topology` | `supervisor.topology` | Set `combined` for the default single supervisor path, or `sidecar` to move pod-level network enforcement and the gateway session into a dedicated sidecar. | +| `sidecar.proxy_uid` | `supervisor.sidecar.proxyUid` | Non-root UID used by the relaxed sidecar when process/binary-aware network policy is disabled. The default binary-aware sidecar runs as UID 0. The network init container exempts the effective sidecar UID from proxy redirection. | +| `sidecar.process_binary_aware_network_policy` | `supervisor.sidecar.processBinaryAwareNetworkPolicy` | Keep process/binary-aware network policy enabled in `sidecar` topology. The default runs the sidecar as UID 0 with `SYS_PTRACE` and `DAC_READ_SEARCH`. Set false to run as `proxy_uid`, drop both capabilities, and enforce endpoint/L7 policy without matching `policy.binaries`. | | `app_armor_profile` | `server.appArmorProfile` | Set the sandbox agent container's AppArmor profile. Helm defaults this to `Unconfined` so AppArmor-enabled nodes do not block supervisor network namespace setup. Set the Helm value to an empty string to omit the field, or use `RuntimeDefault` or `Localhost/` for operator-managed profiles. | | `workspace_default_storage_size` | `server.workspaceDefaultStorageSize` | Set the default workspace PVC size for new sandboxes. | | `sa_token_ttl_secs` | `server.sandboxJwt.k8sSaTokenTtlSecs` | Set the projected ServiceAccount token TTL used for the bootstrap token exchange. | +In `combined` topology, the agent container carries the Linux capabilities +needed by the supervisor for network namespace setup, Landlock filesystem +policy, process privilege changes, and network policy enforcement. In `sidecar` +topology, the agent container runs as the resolved sandbox UID/GID with no added +Linux capabilities. A root init container performs the nftables setup, and the +long-running binary-aware sidecar runs as UID 0, drops default capabilities, +and adds `SYS_PTRACE` plus `DAC_READ_SEARCH` for workload process identity +resolution through shared `/proc`. The +`sidecar.process_binary_aware_network_policy = false` setting runs it as the +configured non-root `proxy_uid`, removes both capabilities, and relaxes network +policy to endpoint/L7 matching only. The +network sidecar owns gateway authentication and writes local policy/provider +state to the process supervisor over a local control socket, so the agent +container does not mount the sandbox bootstrap token or client TLS secret in +the default sidecar path. The provider environment is refreshed by the network +sidecar after settings polls and streamed to the process supervisor so future +child processes can see updated provider env without gateway access in the +agent container. +Sidecar mode keeps gateway session and SSH behavior. The process supervisor +applies Landlock filesystem policy and child seccomp filters where supported, +but it does not perform root-to-sandbox privilege dropping or supervisor +identity mount isolation. Network policy still runs in the sidecar, and sidecar +pods set `shareProcessNamespace: true` so the network sidecar can resolve +process/binary identity through `/proc/`. + The Kubernetes driver creates namespaced `agents.x-k8s.io` `Sandbox` resources from the Kubernetes SIG Apps [agent-sandbox](https://github.com/kubernetes-sigs/agent-sandbox) project. It detects the served Sandbox API at runtime, caches the selected API version for the gateway process, and uses `v1beta1` when available before falling back to `v1alpha1`, so supported Agent Sandbox installations work without version-specific operator configuration. The Agent Sandbox controller turns those resources into sandbox pods and related storage. If Agent Sandbox is upgraded in place, restart the OpenShell gateway after the controller and CRD rollout completes so the gateway can detect the served API versions again. diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index 47b8730dc1..e4f47008c9 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -393,6 +393,21 @@ require_cmd() { fi } +configure_fixture_container_engine() { + [ -n "${CONTAINER_ENGINE:-}" ] || return 0 + local selected_engine + selected_engine="$(printf '%s' "${CONTAINER_ENGINE}" | tr '[:upper:]' '[:lower:]')" + case "${selected_engine}" in + docker|podman) + ;; + *) + echo "ERROR: CONTAINER_ENGINE=${CONTAINER_ENGINE} is invalid; expected docker or podman" >&2 + exit 2 + ;; + esac + export CONTAINER_ENGINE="${selected_engine}" +} + require_cmd helm require_cmd kubectl require_cmd curl @@ -423,6 +438,8 @@ else KUBE_CONTEXT="k3d-${CLUSTER_NAME}" fi +configure_fixture_container_engine + if [ -z "${OPENSHELL_E2E_KUBE_BUILD_IMAGES+x}" ]; then if [ "${CLUSTER_CREATED_BY_US}" = "1" ]; then OPENSHELL_E2E_KUBE_BUILD_IMAGES=1 @@ -501,7 +518,7 @@ if [ -z "${HOST_GATEWAY_IP}" ] \ # is unreachable for the typical test-host listener (0.0.0.0 bind). detected="$(docker network inspect "${net}" \ -f '{{range .IPAM.Config}}{{.Gateway}}{{"\n"}}{{end}}' 2>/dev/null \ - | awk '/^[0-9.]+$/ { print; exit }')" + | awk '/^[0-9.]+$/ { print; exit }' || true)" if [ -n "${detected}" ]; then HOST_GATEWAY_IP="${detected}" echo "Detected host gateway IP ${HOST_GATEWAY_IP} from docker network '${net}'." diff --git a/tasks/helm.toml b/tasks/helm.toml index f25dadb09c..24b6667b1d 100644 --- a/tasks/helm.toml +++ b/tasks/helm.toml @@ -55,16 +55,46 @@ description = "Run skaffold dev for deploy/helm/openshell (iterative deploy)" dir = "deploy/helm/openshell" run = "skaffold dev" +["helm:skaffold:dev:sidecar"] +description = "Run skaffold dev with the Kubernetes supervisor sidecar topology" +dir = "deploy/helm/openshell" +run = "skaffold dev -p sidecar" + +["helm:skaffold:dev:sidecar-mtls"] +description = "Run skaffold dev with the Kubernetes supervisor sidecar topology and TLS/mTLS enabled" +dir = "deploy/helm/openshell" +run = "skaffold dev -p sidecar-mtls" + ["helm:skaffold:run"] description = "Run skaffold run for deploy/helm/openshell (one-shot deploy)" dir = "deploy/helm/openshell" run = "skaffold run" +["helm:skaffold:run:sidecar"] +description = "Run skaffold run with the Kubernetes supervisor sidecar topology" +dir = "deploy/helm/openshell" +run = "skaffold run -p sidecar" + +["helm:skaffold:run:sidecar-mtls"] +description = "Run skaffold run with the Kubernetes supervisor sidecar topology and TLS/mTLS enabled" +dir = "deploy/helm/openshell" +run = "skaffold run -p sidecar-mtls" + ["helm:skaffold:delete"] description = "Run skaffold delete for deploy/helm/openshell" dir = "deploy/helm/openshell" run = "skaffold delete" +["helm:skaffold:delete:sidecar"] +description = "Run skaffold delete for the Kubernetes supervisor sidecar topology" +dir = "deploy/helm/openshell" +run = "skaffold delete -p sidecar" + +["helm:skaffold:delete:sidecar-mtls"] +description = "Run skaffold delete for the Kubernetes supervisor sidecar topology with TLS/mTLS enabled" +dir = "deploy/helm/openshell" +run = "skaffold delete -p sidecar-mtls" + ["helm:skaffold:diagnose"] description = "Run skaffold diagnose for deploy/helm/openshell" dir = "deploy/helm/openshell" diff --git a/tasks/test.toml b/tasks/test.toml index db3878f756..c08fcc5a04 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -114,6 +114,11 @@ run = [ "AGENT_SANDBOX_VERSION=v0.4.6 e2e/rust/e2e-kubernetes.sh", ] +["e2e:kubernetes:sidecar"] +description = "Run Kubernetes e2e with the supervisor sidecar topology overlay" +env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-sidecar.yaml" } +run = "e2e/rust/e2e-kubernetes.sh" + ["e2e:kubernetes:db"] description = "Run Kubernetes e2e with all database backend scenarios (SQLite and external PostgreSQL with existingSecret)" env = { OPENSHELL_E2E_KUBE_DB_SCENARIOS = "1" } From 614c8c164d15024ac44c709c98dcc23bb3350fa5 Mon Sep 17 00:00:00 2001 From: mjamiv <142179942+mjamiv@users.noreply.github.com> Date: Fri, 10 Jul 2026 16:17:37 -0400 Subject: [PATCH 09/13] feat(kubernetes): support PVC subPath driver config (#2034) * feat(kubernetes): support PVC subPath driver config Signed-off-by: mjamiv * test(kubernetes): cover writable PVC driver config Signed-off-by: mjamiv * fix(kubernetes): address PVC subPath review feedback Signed-off-by: mjamiv * fix(kubernetes): address PVC config review follow-up Signed-off-by: mjamiv * fix(kubernetes): address PVC review follow-ups --------- Signed-off-by: mjamiv --- crates/openshell-core/src/driver_mounts.rs | 32 +- crates/openshell-driver-docker/src/tests.rs | 33 + crates/openshell-driver-kubernetes/README.md | 54 + .../openshell-driver-kubernetes/src/driver.rs | 988 +++++++++++++++++- .../openshell-driver-podman/src/container.rs | 34 + docs/reference/sandbox-compute-drivers.mdx | 63 ++ 6 files changed, 1152 insertions(+), 52 deletions(-) diff --git a/crates/openshell-core/src/driver_mounts.rs b/crates/openshell-core/src/driver_mounts.rs index fa43edf533..086d992c37 100644 --- a/crates/openshell-core/src/driver_mounts.rs +++ b/crates/openshell-core/src/driver_mounts.rs @@ -91,6 +91,19 @@ pub fn validate_container_mount_target(target: &str) -> Result<(), String> { if !target.starts_with('/') { return Err("mount target must be an absolute container path".to_string()); } + if target != "/" { + let segments = target.split('/').skip(1).collect::>(); + let has_internal_empty_segment = segments + .iter() + .take(segments.len().saturating_sub(1)) + .any(|segment| segment.is_empty()); + if has_internal_empty_segment || segments.contains(&".") { + return Err( + "mount target must be normalized and must not contain empty path segments or '.'" + .to_string(), + ); + } + } let path = Path::new(target); if path == Path::new("/") { return Err("mount target must not be the container root".to_string()); @@ -122,7 +135,8 @@ pub fn normalize_mount_target(target: &str) -> String { target.trim_end_matches('/').to_string() } -fn path_is_or_under(path: &Path, parent: &Path) -> bool { +/// Return true when `path` is exactly `parent` or is contained below it. +pub fn path_is_or_under(path: &Path, parent: &Path) -> bool { path == parent || path.starts_with(parent) } @@ -185,4 +199,20 @@ mod tests { "mount target must not contain surrounding whitespace" ); } + #[test] + fn mount_target_rejects_internal_empty_or_dot_segments() { + assert_eq!( + validate_container_mount_target("/sandbox/work//tmp").unwrap_err(), + "mount target must be normalized and must not contain empty path segments or '.'" + ); + assert_eq!( + validate_container_mount_target("/sandbox/work/./tmp").unwrap_err(), + "mount target must be normalized and must not contain empty path segments or '.'" + ); + assert_eq!( + validate_container_mount_target("/sandbox/work/../../tmp").unwrap_err(), + "mount target must not contain '..'" + ); + validate_container_mount_target("/sandbox/work/").unwrap(); + } } diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 560d47de7a..594308bce5 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -755,6 +755,39 @@ fn driver_config_allows_explicit_writable_volume_mounts() { assert_eq!(mounts[0].read_only, Some(false)); } +#[test] +fn driver_config_rejects_duplicate_mount_targets() { + let mut sandbox = test_sandbox(); + sandbox + .spec + .as_mut() + .unwrap() + .template + .as_mut() + .unwrap() + .driver_config = Some(json_struct(serde_json::json!({ + "mounts": [ + { + "type": "volume", + "source": "work-nfs", + "target": "/sandbox/work" + }, + { + "type": "tmpfs", + "target": "/sandbox/work" + } + ] + }))); + + let err = build_container_create_body(&sandbox, &runtime_config()).unwrap_err(); + + assert_eq!(err.code(), tonic::Code::FailedPrecondition); + assert!( + err.message() + .contains("duplicate docker driver_config mount target") + ); +} + #[test] fn driver_config_rejects_bind_mounts_unless_enabled() { let mut sandbox = test_sandbox(); diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index 688660dbb5..96e54ad448 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -117,6 +117,13 @@ nested schema and currently accepts: - `pod.priority_class_name` - `containers.agent.resources.requests` - `containers.agent.resources.limits` +- `containers.agent.volume_mounts[].name` +- `containers.agent.volume_mounts[].mount_path` +- `containers.agent.volume_mounts[].sub_path` +- `containers.agent.volume_mounts[].read_only` +- `volumes[].name` +- `volumes[].persistent_volume_claim.claim_name` +- `volumes[].persistent_volume_claim.read_only` Nested keys inside the `kubernetes` block use snake_case. The top-level `driver_config` envelope is keyed by driver names, so `kubernetes` is not part @@ -139,3 +146,50 @@ driver's configured `default_runtime_class_name`; the typed public public `--gpu` flag for the default GPU request, pass a count to `--gpu` for counted GPU requests, and use `driver_config` only for additional driver-owned resource details. + +Use PVC volumes to mount existing Kubernetes PersistentVolumeClaims into the +agent container. PVC volumes and mounts default to read-only unless +`read_only: false` is set explicitly. Read-write access requires +`read_only: false` on both the PVC volume and each writable mount. The driver +rejects duplicate volume names, invalid DNS-1123 volume labels or PVC claim +subdomain names, mounts that reference unknown volumes, non-normalized or +protected mount paths, and absolute or parent-traversing `sub_path` values. + +Any explicit driver-config mount under `/sandbox` disables the driver's +default `/sandbox` workspace PVC injection for that sandbox. Only the explicit +mount paths persist through the external PVC; other `/sandbox` paths come from +the current sandbox image. + +```shell +openshell sandbox create \ + --driver-config-json '{ + "kubernetes": { + "volumes": [{ + "name": "user-data", + "persistent_volume_claim": { + "claim_name": "pvc-user-data-123", + "read_only": false + } + }], + "containers": { + "agent": { + "volume_mounts": [ + { + "name": "user-data", + "mount_path": "/sandbox/.openshell/workspace", + "sub_path": "workspace", + "read_only": false + }, + { + "name": "user-data", + "mount_path": "/sandbox/.openshell/memory", + "sub_path": "memory", + "read_only": false + } + ] + } + } + } + }' \ + -- claude +``` diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index db7492d27d..5315d30409 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -10,12 +10,15 @@ use crate::config::{ SupervisorTopology, }; use futures::{Stream, StreamExt, TryStreamExt}; -use k8s_openapi::api::core::v1::{Event as KubeEventObj, Namespace, Node}; +use k8s_openapi::api::core::v1::{ + Event as KubeEventObj, Namespace, Node, PersistentVolumeClaimVolumeSource, Volume, VolumeMount, +}; use kube::api::{Api, ApiResource, DeleteParams, ListParams, PostParams}; use kube::core::gvk::GroupVersionKind; use kube::core::{DynamicObject, ObjectMeta}; use kube::runtime::watcher::{self, Event}; use kube::{Client, Error as KubeError}; +use openshell_core::driver_mounts; use openshell_core::driver_utils::{ LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, SUPERVISOR_IMAGE_BINARY_PATH, }; @@ -34,7 +37,8 @@ use openshell_core::proto::compute::v1::{ }; use openshell_core::proto_struct::{struct_to_json_object, value_to_json}; use serde::Deserialize; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashSet}; +use std::path::Path; use std::pin::Pin; use std::sync::Arc; use std::time::Duration; @@ -107,29 +111,39 @@ struct AgentSandboxApi { struct KubernetesSandboxDriverConfig { pod: KubernetesPodDriverConfig, containers: KubernetesDriverContainersConfig, + volumes: Vec, } impl KubernetesSandboxDriverConfig { - fn from_sandbox(sandbox: &Sandbox) -> Result { - let Some(template) = sandbox - .spec - .as_ref() - .and_then(|spec| spec.template.as_ref()) - else { - return Ok(Self::default()); - }; - - Self::from_template(template) - } - fn from_template(template: &SandboxTemplate) -> Result { let Some(config) = template.driver_config.as_ref() else { return Ok(Self::default()); }; let json = serde_json::Value::Object(struct_to_json_object(config)); - serde_json::from_value(json) - .map_err(|err| format!("invalid kubernetes driver_config: {err}")) + let config: Self = serde_json::from_value(json) + .map_err(|err| format!("invalid kubernetes driver_config: {err}"))?; + config + .validate() + .map_err(|err| format!("invalid kubernetes driver_config: {err}"))?; + Ok(config) + } + + fn validate(&self) -> Result<(), String> { + validate_kubernetes_driver_volumes(&self.volumes)?; + validate_kubernetes_driver_volume_mounts( + &self.volumes, + &self.containers.agent.volume_mounts, + ) + } + + fn has_explicit_sandbox_data_mount(&self) -> bool { + self.containers.agent.volume_mounts.iter().any(|mount| { + driver_mounts::path_is_or_under( + Path::new(&mount.mount_path), + Path::new(WORKSPACE_MOUNT_PATH), + ) + }) } } @@ -152,6 +166,7 @@ struct KubernetesDriverContainersConfig { #[serde(default, deny_unknown_fields)] struct KubernetesContainerDriverConfig { resources: KubernetesContainerResourceConfig, + volume_mounts: Vec, } #[derive(Debug, Clone, Default, Deserialize)] @@ -161,6 +176,220 @@ struct KubernetesContainerResourceConfig { limits: BTreeMap, } +#[derive(Debug, Clone, Default, Deserialize)] +#[serde(default, deny_unknown_fields)] +struct KubernetesDriverVolumeConfig { + name: String, + persistent_volume_claim: KubernetesPersistentVolumeClaimConfig, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(default, deny_unknown_fields)] +struct KubernetesPersistentVolumeClaimConfig { + claim_name: String, + read_only: bool, +} + +impl Default for KubernetesPersistentVolumeClaimConfig { + fn default() -> Self { + Self { + claim_name: String::new(), + read_only: true, + } + } +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(default, deny_unknown_fields)] +struct KubernetesDriverVolumeMountConfig { + name: String, + mount_path: String, + sub_path: Option, + read_only: bool, +} + +impl Default for KubernetesDriverVolumeMountConfig { + fn default() -> Self { + Self { + name: String::new(), + mount_path: String::new(), + sub_path: None, + read_only: true, + } + } +} + +impl From<&KubernetesDriverVolumeConfig> for Volume { + fn from(volume: &KubernetesDriverVolumeConfig) -> Self { + Self { + name: volume.name.clone(), + persistent_volume_claim: Some(PersistentVolumeClaimVolumeSource { + claim_name: volume.persistent_volume_claim.claim_name.clone(), + read_only: Some(volume.persistent_volume_claim.read_only), + }), + ..Default::default() + } + } +} + +impl From<&KubernetesDriverVolumeMountConfig> for VolumeMount { + fn from(mount: &KubernetesDriverVolumeMountConfig) -> Self { + Self { + name: mount.name.clone(), + mount_path: mount.mount_path.clone(), + read_only: Some(mount.read_only), + sub_path: mount.sub_path.clone(), + ..Default::default() + } + } +} + +const CLIENT_TLS_VOLUME_NAME: &str = "openshell-client-tls"; +const SERVICE_ACCOUNT_TOKEN_VOLUME_NAME: &str = "openshell-sa-token"; +const SERVICE_ACCOUNT_TOKEN_MOUNT_PATH: &str = "/var/run/secrets/openshell"; + +const KUBERNETES_DRIVER_RESERVED_VOLUME_NAMES: &[&str] = &[ + CLIENT_TLS_VOLUME_NAME, + SERVICE_ACCOUNT_TOKEN_VOLUME_NAME, + SPIFFE_WORKLOAD_API_VOLUME_NAME, + SUPERVISOR_VOLUME_NAME, + WORKSPACE_VOLUME_NAME, +]; + +const KUBERNETES_DRIVER_PROTECTED_MOUNT_PATHS: &[&str] = &[SERVICE_ACCOUNT_TOKEN_MOUNT_PATH]; + +fn validate_kubernetes_driver_volumes( + volumes: &[KubernetesDriverVolumeConfig], +) -> Result<(), String> { + let mut names = HashSet::new(); + for volume in volumes { + validate_kubernetes_dns1123_label(&volume.name, "volumes[].name")?; + let name = volume.name.as_str(); + if KUBERNETES_DRIVER_RESERVED_VOLUME_NAMES.contains(&name) { + return Err(format!( + "volume name '{name}' is reserved for OpenShell-managed volumes" + )); + } + if !names.insert(name) { + return Err(format!( + "duplicate kubernetes driver_config volume '{name}'" + )); + } + validate_kubernetes_dns1123_subdomain( + &volume.persistent_volume_claim.claim_name, + "volumes[].persistent_volume_claim.claim_name", + )?; + } + Ok(()) +} + +fn validate_kubernetes_driver_volume_mounts( + volumes: &[KubernetesDriverVolumeConfig], + volume_mounts: &[KubernetesDriverVolumeMountConfig], +) -> Result<(), String> { + let mut volume_read_only = BTreeMap::new(); + for volume in volumes { + volume_read_only.insert( + volume.name.as_str(), + volume.persistent_volume_claim.read_only, + ); + } + + let mut mount_paths = HashSet::new(); + for mount in volume_mounts { + validate_kubernetes_dns1123_label(&mount.name, "containers.agent.volume_mounts[].name")?; + let volume_name = mount.name.as_str(); + let Some(volume_is_read_only) = volume_read_only.get(volume_name) else { + return Err(format!( + "volume mount references unknown kubernetes driver_config volume '{volume_name}'" + )); + }; + if *volume_is_read_only && !mount.read_only { + return Err(format!( + "volume mount '{volume_name}' cannot set read_only=false because the PVC volume is read_only=true" + )); + } + + driver_mounts::validate_container_mount_target(&mount.mount_path)?; + let normalized_mount_path = driver_mounts::normalize_mount_target(&mount.mount_path); + if !mount_paths.insert(normalized_mount_path.clone()) { + return Err(format!( + "duplicate kubernetes driver_config mount target '{normalized_mount_path}'" + )); + } + + if let Some(sub_path) = mount.sub_path.as_ref() { + driver_mounts::validate_mount_subpath(sub_path)?; + } + } + Ok(()) +} + +// TODO: replace with an openshell_core Kubernetes-name helper once available. +fn is_dns_label(label: &str) -> bool { + if label.is_empty() || label.len() > 63 || label.starts_with('-') || label.ends_with('-') { + return false; + } + label + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') +} + +// TODO: replace with an openshell_core Kubernetes-name helper once available. +fn is_dns_subdomain(value: &str) -> bool { + value.len() <= 253 && value.split('.').all(is_dns_label) +} + +fn validate_kubernetes_dns1123_label(value: &str, field: &str) -> Result<(), String> { + if !is_dns_label(value) { + return Err(format!( + "{field} must be a DNS-1123 label: use lowercase alphanumeric characters or '-', start and end with an alphanumeric character, and use at most 63 characters" + )); + } + Ok(()) +} + +fn validate_kubernetes_dns1123_subdomain(value: &str, field: &str) -> Result<(), String> { + if !is_dns_subdomain(value) { + return Err(format!( + "{field} must be a DNS-1123 subdomain: use lowercase alphanumeric characters, '-' or '.', start and end with an alphanumeric character, and use at most 253 characters" + )); + } + Ok(()) +} + +fn mount_path_conflicts_with_protected_path(mount_path: &str, protected_path: &str) -> bool { + driver_mounts::path_is_or_under(Path::new(mount_path), Path::new(protected_path)) + || driver_mounts::path_is_or_under(Path::new(protected_path), Path::new(mount_path)) +} + +fn validate_kubernetes_protected_path_conflicts( + volume_mounts: &[KubernetesDriverVolumeMountConfig], + protected_paths: &[&str], +) -> Result<(), String> { + for mount in volume_mounts { + let mount_path = mount.mount_path.as_str(); + for protected_path in protected_paths { + if mount_path_conflicts_with_protected_path(mount_path, protected_path) { + return Err(format!( + "mount path '{mount_path}' conflicts with reserved OpenShell path '{protected_path}'" + )); + } + } + } + Ok(()) +} + +fn kubernetes_driver_volume_to_k8s(volume: &KubernetesDriverVolumeConfig) -> serde_json::Value { + serde_json::to_value(Volume::from(volume)).expect("Volume serializes to JSON") +} + +fn kubernetes_driver_volume_mount_to_k8s( + mount: &KubernetesDriverVolumeMountConfig, +) -> serde_json::Value { + serde_json::to_value(VolumeMount::from(mount)).expect("VolumeMount serializes to JSON") +} + // --------------------------------------------------------------------------- // Default workspace persistence (temporary — will be replaced by snapshotting) // --------------------------------------------------------------------------- @@ -274,6 +503,20 @@ impl KubernetesComputeDriver { &self.config.ssh_socket_path } + fn validate_driver_config_for_sandbox( + &self, + sandbox: &Sandbox, + ) -> Result { + kubernetes_driver_config_for_spec( + sandbox.spec.as_ref(), + self.config.provider_spiffe_enabled().then_some( + self.config + .provider_spiffe_workload_api_socket_path + .as_str(), + ), + ) + } + fn agent_sandbox_api(&self, client: Client, sandbox_api_version: &str) -> AgentSandboxApi { let gvk = GroupVersionKind::gvk(SANDBOX_GROUP, sandbox_api_version, SANDBOX_KIND); let resource = ApiResource::from_gvk(&gvk); @@ -421,7 +664,8 @@ impl KubernetesComputeDriver { } pub async fn validate_sandbox_create(&self, sandbox: &Sandbox) -> Result<(), tonic::Status> { - let _ = KubernetesSandboxDriverConfig::from_sandbox(sandbox) + let _ = self + .validate_driver_config_for_sandbox(sandbox) .map_err(tonic::Status::invalid_argument)?; let gpu_requirements = sandbox .spec @@ -530,8 +774,6 @@ impl KubernetesComputeDriver { #[allow(clippy::similar_names)] pub async fn create_sandbox(&self, sandbox: &Sandbox) -> Result<(), KubernetesDriverError> { - let _ = KubernetesSandboxDriverConfig::from_sandbox(sandbox) - .map_err(KubernetesDriverError::InvalidArgument)?; let gpu_requirements = sandbox .spec .as_ref() @@ -590,6 +832,8 @@ impl KubernetesComputeDriver { }; validate_sidecar_proxy_identity(¶ms)?; + let data = sandbox_to_k8s_spec(sandbox.spec.as_ref(), ¶ms) + .map_err(KubernetesDriverError::InvalidArgument)?; let mut obj = DynamicObject::new(name, &agent_sandbox_api.resource); // Copy only the SCC-related annotations onto the Sandbox CR for // traceability. Copying the full namespace annotation map exposes @@ -617,7 +861,7 @@ impl KubernetesComputeDriver { ..Default::default() }; - obj.data = sandbox_to_k8s_spec(sandbox.spec.as_ref(), ¶ms); + obj.data = data; match tokio::time::timeout( KUBE_API_TIMEOUT, agent_sandbox_api.api.create(&PostParams::default(), &obj), @@ -1893,27 +2137,55 @@ fn spec_pod_env(spec: Option<&SandboxSpec>) -> std::collections::HashMap KubernetesSandboxDriverConfig { - KubernetesSandboxDriverConfig::from_template(template) - .expect("validated Kubernetes driver_config") +fn kubernetes_driver_config_for_spec( + spec: Option<&SandboxSpec>, + provider_spiffe_workload_api_socket_path: Option<&str>, +) -> Result { + let config = spec + .and_then(|spec| spec.template.as_ref()) + .map(KubernetesSandboxDriverConfig::from_template) + .transpose()? + .unwrap_or_default(); + let mut protected_paths = KUBERNETES_DRIVER_PROTECTED_MOUNT_PATHS.to_vec(); + let provider_spiffe_mount_path; + if let Some(socket_path) = provider_spiffe_workload_api_socket_path { + provider_spiffe_mount_path = spiffe_socket_mount_path(socket_path); + protected_paths.push(&provider_spiffe_mount_path); + } + validate_kubernetes_protected_path_conflicts( + &config.containers.agent.volume_mounts, + &protected_paths, + )?; + Ok(config) } fn sandbox_to_k8s_spec( spec: Option<&SandboxSpec>, params: &SandboxPodParams<'_>, -) -> serde_json::Value { +) -> Result { + let driver_config = + kubernetes_driver_config_for_spec(spec, provider_spiffe_socket_path(params))?; let mut root = serde_json::Map::new(); + // Determine early whether OpenShell should inject its default workspace + // PVC. Explicit Kubernetes driver-config mounts under /sandbox/ take + // ownership of workspace persistence. + // We need this flag before building the podTemplate because the workspace + // persistence transforms are applied inside sandbox_template_to_k8s. + let user_has_explicit_workspace_mount = driver_config.has_explicit_sandbox_data_mount(); + let inject_workspace = !user_has_explicit_workspace_mount; + if let Some(spec) = spec { let pod_env = spec_pod_env(Some(spec)); if let Some(template) = spec.template.as_ref() { root.insert( "podTemplate".to_string(), - sandbox_template_to_k8s_with_gpu_requirements( + sandbox_template_to_k8s_with_validated_config( template, driver_gpu_requirements(spec.resource_requirements.as_ref()), &pod_env, - true, + &driver_config, + inject_workspace, params, ), ); @@ -1926,29 +2198,32 @@ fn sandbox_to_k8s_spec( } } - root.insert( - "volumeClaimTemplates".to_string(), - default_workspace_volume_claim_templates(params.workspace_default_storage_size), - ); + if inject_workspace { + root.insert( + "volumeClaimTemplates".to_string(), + default_workspace_volume_claim_templates(params.workspace_default_storage_size), + ); + } // podTemplate is required by the Kubernetes CRD - ensure it's always present if !root.contains_key("podTemplate") { let pod_env = spec_pod_env(spec); root.insert( "podTemplate".to_string(), - sandbox_template_to_k8s_with_gpu_requirements( + sandbox_template_to_k8s_with_validated_config( &SandboxTemplate::default(), driver_gpu_requirements(spec.and_then(|s| s.resource_requirements.as_ref())), &pod_env, - true, + &driver_config, + inject_workspace, params, ), ); } - serde_json::Value::Object( + Ok(serde_json::Value::Object( std::iter::once(("spec".to_string(), serde_json::Value::Object(root))).collect(), - ) + )) } #[cfg(test)] @@ -1960,15 +2235,19 @@ fn sandbox_template_to_k8s( params: &SandboxPodParams<'_>, ) -> serde_json::Value { let gpu_requirements = gpu.then_some(GpuResourceRequirements { count: None }); - sandbox_template_to_k8s_with_gpu_requirements( + let driver_config = KubernetesSandboxDriverConfig::from_template(template) + .expect("test Kubernetes driver_config should be valid"); + sandbox_template_to_k8s_with_validated_config( template, gpu_requirements.as_ref(), spec_environment, + &driver_config, inject_workspace, params, ) } +#[cfg(test)] fn sandbox_template_to_k8s_with_gpu_requirements( template: &SandboxTemplate, gpu_requirements: Option<&GpuResourceRequirements>, @@ -1976,8 +2255,26 @@ fn sandbox_template_to_k8s_with_gpu_requirements( inject_workspace: bool, params: &SandboxPodParams<'_>, ) -> serde_json::Value { - let driver_config = kubernetes_driver_config(template); + let driver_config = KubernetesSandboxDriverConfig::from_template(template) + .expect("test Kubernetes driver_config should be valid"); + sandbox_template_to_k8s_with_validated_config( + template, + gpu_requirements, + spec_environment, + &driver_config, + inject_workspace, + params, + ) +} +fn sandbox_template_to_k8s_with_validated_config( + template: &SandboxTemplate, + gpu_requirements: Option<&GpuResourceRequirements>, + spec_environment: &std::collections::HashMap, + driver_config: &KubernetesSandboxDriverConfig, + inject_workspace: bool, + params: &SandboxPodParams<'_>, +) -> serde_json::Value { let mut metadata = serde_json::Map::new(); let mut pod_labels = template .labels @@ -2141,7 +2438,7 @@ fn sandbox_template_to_k8s_with_gpu_requirements( let mut volume_mounts: Vec = Vec::new(); if !params.client_tls_secret_name.is_empty() { volume_mounts.push(serde_json::json!({ - "name": "openshell-client-tls", + "name": CLIENT_TLS_VOLUME_NAME, "mountPath": "/etc/openshell-tls/client", "readOnly": true })); @@ -2154,10 +2451,18 @@ fn sandbox_template_to_k8s_with_gpu_requirements( })); } volume_mounts.push(serde_json::json!({ - "name": "openshell-sa-token", - "mountPath": "/var/run/secrets/openshell", + "name": SERVICE_ACCOUNT_TOKEN_VOLUME_NAME, + "mountPath": SERVICE_ACCOUNT_TOKEN_MOUNT_PATH, "readOnly": true, })); + volume_mounts.extend( + driver_config + .containers + .agent + .volume_mounts + .iter() + .map(kubernetes_driver_volume_mount_to_k8s), + ); container.insert( "volumeMounts".to_string(), serde_json::Value::Array(volume_mounts), @@ -2183,7 +2488,7 @@ fn sandbox_template_to_k8s_with_gpu_requirements( SupervisorTopology::Sidecar => 0o440, }; volumes.push(serde_json::json!({ - "name": "openshell-client-tls", + "name": CLIENT_TLS_VOLUME_NAME, "secret": { "secretName": params.client_tls_secret_name, "defaultMode": client_tls_default_mode @@ -2209,7 +2514,7 @@ fn sandbox_template_to_k8s_with_gpu_requirements( SupervisorTopology::Sidecar => 0o440, }; volumes.push(serde_json::json!({ - "name": "openshell-sa-token", + "name": SERVICE_ACCOUNT_TOKEN_VOLUME_NAME, "projected": { "sources": [{ "serviceAccountToken": { @@ -2221,6 +2526,12 @@ fn sandbox_template_to_k8s_with_gpu_requirements( "defaultMode": sa_token_default_mode } })); + volumes.extend( + driver_config + .volumes + .iter() + .map(kubernetes_driver_volume_to_k8s), + ); spec.insert("volumes".to_string(), serde_json::Value::Array(volumes)); // Add hostAliases so sandbox pods can reach the Docker host. @@ -2264,8 +2575,8 @@ fn sandbox_template_to_k8s_with_gpu_requirements( } // Inject workspace persistence (init container + PVC volume mount) so - // that /sandbox data survives pod rescheduling. Skipped when the user - // provides custom volumeClaimTemplates to avoid conflicts. + // that /sandbox data survives pod rescheduling. Skipped when the user + // provides custom storage through driver_config. if inject_workspace { apply_workspace_persistence( &mut result, @@ -2559,9 +2870,9 @@ fn provider_spiffe_socket_path<'a>(params: &'a SandboxPodParams<'a>) -> Option<& } fn spiffe_socket_mount_path(socket_path: &str) -> String { - std::path::Path::new(socket_path) + Path::new(socket_path) .parent() - .and_then(std::path::Path::to_str) + .and_then(Path::to_str) .filter(|path| !path.is_empty() && *path != "/") .expect("provider SPIFFE socket path should be validated before pod rendering") .to_string() @@ -2733,6 +3044,13 @@ mod tests { } } + fn sandbox_to_k8s_spec_for_test( + spec: Option<&SandboxSpec>, + params: &SandboxPodParams<'_>, + ) -> serde_json::Value { + sandbox_to_k8s_spec(spec, params).expect("test Kubernetes driver_config should be valid") + } + fn kube_api_error(code: u16, message: &str) -> KubeError { KubeError::Api(kube::core::ErrorResponse { status: if code == 404 { @@ -2799,7 +3117,7 @@ mod tests { } #[test] - fn driver_config_from_sandbox_rejects_unknown_fields() { + fn driver_config_for_spec_rejects_unknown_fields() { let sandbox = Sandbox { id: "sandbox-123".to_string(), spec: Some(SandboxSpec { @@ -2814,11 +3132,579 @@ mod tests { ..Default::default() }; - let err = KubernetesSandboxDriverConfig::from_sandbox(&sandbox).unwrap_err(); + let err = kubernetes_driver_config_for_spec(sandbox.spec.as_ref(), None).unwrap_err(); assert!(err.contains("unknown field")); assert!(err.contains("gpu_device_ids")); } + #[test] + fn driver_config_pvc_subpath_mounts_render_in_pod_template() { + let template = SandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "volumes": [{ + "name": "user-data", + "persistent_volume_claim": { + "claim_name": "pvc-user-data-123", + "read_only": false + } + }], + "containers": { + "agent": { + "volume_mounts": [ + { + "name": "user-data", + "mount_path": "/sandbox/.openshell/workspace", + "sub_path": "workspace", + "read_only": false + }, + { + "name": "user-data", + "mount_path": "/sandbox/.openshell/memory", + "sub_path": "memory" + } + ] + } + } + }))), + ..SandboxTemplate::default() + }; + let spec = SandboxSpec { + template: Some(template), + ..SandboxSpec::default() + }; + + let cr = sandbox_to_k8s_spec_for_test(Some(&spec), &SandboxPodParams::default()); + let pod_template = &cr["spec"]["podTemplate"]; + + let volumes = pod_template["spec"]["volumes"] + .as_array() + .expect("volumes should exist"); + let user_volume = volumes + .iter() + .find(|volume| volume["name"] == "user-data") + .expect("user PVC volume should be rendered"); + assert_eq!( + user_volume["persistentVolumeClaim"]["claimName"], + "pvc-user-data-123" + ); + assert_eq!(user_volume["persistentVolumeClaim"]["readOnly"], false); + + let mounts = pod_template["spec"]["containers"][0]["volumeMounts"] + .as_array() + .expect("volumeMounts should exist"); + let workspace_mount = mounts + .iter() + .find(|mount| mount["mountPath"] == "/sandbox/.openshell/workspace") + .expect("workspace subPath mount should be rendered"); + assert_eq!(workspace_mount["name"], "user-data"); + assert_eq!(workspace_mount["subPath"], "workspace"); + assert_eq!(workspace_mount["readOnly"], false); + + let memory_mount = mounts + .iter() + .find(|mount| mount["mountPath"] == "/sandbox/.openshell/memory") + .expect("memory subPath mount should be rendered"); + assert_eq!(memory_mount["name"], "user-data"); + assert_eq!(memory_mount["subPath"], "memory"); + assert_eq!(memory_mount["readOnly"], true); + + let spec_obj = cr["spec"].as_object().expect("spec should be an object"); + assert!( + !spec_obj.contains_key("volumeClaimTemplates"), + "explicit /sandbox driver_config mounts should skip the default workspace VCT" + ); + let has_workspace_init = pod_template["spec"]["initContainers"] + .as_array() + .is_some_and(|containers| { + containers + .iter() + .any(|container| container["name"] == WORKSPACE_INIT_CONTAINER_NAME) + }); + assert!( + !has_workspace_init, + "explicit /sandbox driver_config mounts should skip the default workspace init container" + ); + } + + #[test] + fn driver_config_accepts_read_write_pvc_with_multiple_subpath_mounts() { + let template = SandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "volumes": [{ + "name": "user-data", + "persistent_volume_claim": { + "claim_name": "pvc-user-data", + "read_only": false + } + }], + "containers": { + "agent": { + "volume_mounts": [ + { + "name": "user-data", + "mount_path": "/sandbox/.openshell/workspace", + "sub_path": "workspace", + "read_only": false + }, + { + "name": "user-data", + "mount_path": "/sandbox/.openshell/memory", + "sub_path": "memory", + "read_only": false + }, + { + "name": "user-data", + "mount_path": "/sandbox/.openshell/sessions", + "sub_path": "sessions", + "read_only": false + } + ] + } + } + }))), + ..SandboxTemplate::default() + }; + + let config = KubernetesSandboxDriverConfig::from_template(&template) + .expect("read-write PVC with multiple subPath mounts should validate"); + + assert_eq!(config.volumes.len(), 1); + assert_eq!(config.volumes[0].name, "user-data"); + assert_eq!( + config.volumes[0].persistent_volume_claim.claim_name, + "pvc-user-data" + ); + assert!(!config.volumes[0].persistent_volume_claim.read_only); + assert_eq!(config.containers.agent.volume_mounts.len(), 3); + assert!( + config + .containers + .agent + .volume_mounts + .iter() + .all(|mount| !mount.read_only) + ); + assert!(config.has_explicit_sandbox_data_mount()); + } + + #[test] + fn driver_config_rejects_duplicate_pvc_volume_names() { + let template = SandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "volumes": [ + { + "name": "user-data", + "persistent_volume_claim": {"claim_name": "pvc-a"} + }, + { + "name": "user-data", + "persistent_volume_claim": {"claim_name": "pvc-b"} + } + ] + }))), + ..SandboxTemplate::default() + }; + + let err = KubernetesSandboxDriverConfig::from_template(&template).unwrap_err(); + + assert!(err.contains("duplicate kubernetes driver_config volume")); + } + + #[test] + fn driver_config_rejects_duplicate_pvc_volume_mount_targets() { + let template = SandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "volumes": [{ + "name": "user-data", + "persistent_volume_claim": {"claim_name": "pvc-user-data"} + }], + "containers": { + "agent": { + "volume_mounts": [ + { + "name": "user-data", + "mount_path": "/sandbox/.openshell/workspace" + }, + { + "name": "user-data", + "mount_path": "/sandbox/.openshell/workspace" + } + ] + } + } + }))), + ..SandboxTemplate::default() + }; + + let err = KubernetesSandboxDriverConfig::from_template(&template).unwrap_err(); + + assert!(err.contains("duplicate kubernetes driver_config mount target")); + } + + #[test] + fn driver_config_accepts_dns1123_subdomain_pvc_claim_name() { + let template = SandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "volumes": [{ + "name": "user-data", + "persistent_volume_claim": {"claim_name": "pvc.user-data.123"} + }] + }))), + ..SandboxTemplate::default() + }; + + let config = KubernetesSandboxDriverConfig::from_template(&template) + .expect("DNS-1123 subdomain PVC names should validate"); + + assert_eq!( + config.volumes[0].persistent_volume_claim.claim_name, + "pvc.user-data.123" + ); + } + + #[test] + fn driver_config_rejects_invalid_volume_label_and_claim_name() { + for (field, config) in [ + ( + "volumes[].name", + serde_json::json!({ + "volumes": [{ + "name": "User_Data", + "persistent_volume_claim": {"claim_name": "pvc-user-data"} + }] + }), + ), + ( + "volumes[].persistent_volume_claim.claim_name", + serde_json::json!({ + "volumes": [{ + "name": "user-data", + "persistent_volume_claim": {"claim_name": "Pvc_User_Data"} + }] + }), + ), + ] { + let template = SandboxTemplate { + driver_config: Some(json_struct(config)), + ..SandboxTemplate::default() + }; + + let err = KubernetesSandboxDriverConfig::from_template(&template).unwrap_err(); + assert!( + err.contains(field) && err.contains("DNS-1123"), + "expected invalid {field} to fail DNS-1123 validation, got {err}" + ); + } + } + + #[test] + fn driver_config_rejects_mounts_referencing_unknown_volumes() { + let template = SandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "volumes": [{ + "name": "known-data", + "persistent_volume_claim": {"claim_name": "pvc-known"} + }], + "containers": { + "agent": { + "volume_mounts": [{ + "name": "missing-data", + "mount_path": "/sandbox/.openshell/workspace", + "sub_path": "workspace" + }] + } + } + }))), + ..SandboxTemplate::default() + }; + + let err = KubernetesSandboxDriverConfig::from_template(&template).unwrap_err(); + + assert!(err.contains("unknown kubernetes driver_config volume 'missing-data'")); + } + + #[test] + fn driver_config_rejects_shared_reserved_mount_targets() { + for mount_path in [ + "/", + "/sandbox", + "/etc/openshell", + "/etc/openshell-tls/client", + "/opt/openshell/bin", + ] { + let template = SandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "volumes": [{ + "name": "user-data", + "persistent_volume_claim": {"claim_name": "pvc-user-data"} + }], + "containers": { + "agent": { + "volume_mounts": [{ + "name": "user-data", + "mount_path": mount_path + }] + } + } + }))), + ..SandboxTemplate::default() + }; + + let err = KubernetesSandboxDriverConfig::from_template(&template).unwrap_err(); + assert!( + err.contains("mount path") || err.contains("mount target"), + "expected protected mount target {mount_path:?} to be rejected, got {err}" + ); + } + } + + #[test] + fn driver_config_rejects_kubernetes_static_protected_mount_targets() { + let spec = SandboxSpec { + template: Some(SandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "volumes": [{ + "name": "user-data", + "persistent_volume_claim": {"claim_name": "pvc-user-data"} + }], + "containers": { + "agent": { + "volume_mounts": [{ + "name": "user-data", + "mount_path": "/var/run/secrets/openshell" + }] + } + } + }))), + ..SandboxTemplate::default() + }), + ..SandboxSpec::default() + }; + + let err = kubernetes_driver_config_for_spec(Some(&spec), None).unwrap_err(); + + assert!(err.contains("/var/run/secrets/openshell")); + } + + #[test] + fn driver_config_allows_spiffe_workload_path_without_provider_spiffe() { + let spec = SandboxSpec { + template: Some(SandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "volumes": [{ + "name": "user-data", + "persistent_volume_claim": {"claim_name": "pvc-user-data"} + }], + "containers": { + "agent": { + "volume_mounts": [{ + "name": "user-data", + "mount_path": "/spiffe-workload-api" + }] + } + } + }))), + ..SandboxTemplate::default() + }), + ..SandboxSpec::default() + }; + + kubernetes_driver_config_for_spec(Some(&spec), None) + .expect("SPIFFE workload path should only be protected when SPIFFE is enabled"); + } + + #[test] + fn driver_config_rejects_invalid_kubernetes_sub_paths() { + for sub_path in ["/workspace", "../workspace"] { + let template = SandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "volumes": [{ + "name": "user-data", + "persistent_volume_claim": {"claim_name": "pvc-user-data"} + }], + "containers": { + "agent": { + "volume_mounts": [{ + "name": "user-data", + "mount_path": "/sandbox/.openshell/workspace", + "sub_path": sub_path + }] + } + } + }))), + ..SandboxTemplate::default() + }; + + let err = KubernetesSandboxDriverConfig::from_template(&template).unwrap_err(); + assert!( + err.contains("mount subpath must be relative"), + "expected invalid sub_path {sub_path:?} to be rejected, got {err}" + ); + } + } + + #[test] + fn driver_config_defaults_pvc_mounts_to_read_only() { + let template = SandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "volumes": [{ + "name": "user-data", + "persistent_volume_claim": {"claim_name": "pvc-user-data"} + }], + "containers": { + "agent": { + "volume_mounts": [{ + "name": "user-data", + "mount_path": "/sandbox/.openshell/workspace", + "sub_path": "workspace" + }] + } + } + }))), + ..SandboxTemplate::default() + }; + + let pod_template = sandbox_template_to_k8s( + &template, + false, + &std::collections::HashMap::new(), + false, + &SandboxPodParams::default(), + ); + + let volume = pod_template["spec"]["volumes"] + .as_array() + .expect("volumes should exist") + .iter() + .find(|volume| volume["name"] == "user-data") + .expect("user volume should exist"); + assert_eq!(volume["persistentVolumeClaim"]["readOnly"], true); + + let mount = pod_template["spec"]["containers"][0]["volumeMounts"] + .as_array() + .expect("volumeMounts should exist") + .iter() + .find(|mount| mount["mountPath"] == "/sandbox/.openshell/workspace") + .expect("user mount should exist"); + assert_eq!(mount["readOnly"], true); + } + + #[test] + fn driver_config_rejects_read_write_mount_for_read_only_pvc_volume() { + let template = SandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "volumes": [{ + "name": "user-data", + "persistent_volume_claim": { + "claim_name": "pvc-user-data", + "read_only": true + } + }], + "containers": { + "agent": { + "volume_mounts": [{ + "name": "user-data", + "mount_path": "/sandbox/.openshell/workspace", + "read_only": false + }] + } + } + }))), + ..SandboxTemplate::default() + }; + + let err = KubernetesSandboxDriverConfig::from_template(&template).unwrap_err(); + + assert!(err.contains("cannot set read_only=false")); + } + + #[test] + fn driver_config_rejects_reserved_kubernetes_volume_names() { + for volume_name in [ + CLIENT_TLS_VOLUME_NAME, + SERVICE_ACCOUNT_TOKEN_VOLUME_NAME, + SPIFFE_WORKLOAD_API_VOLUME_NAME, + SUPERVISOR_VOLUME_NAME, + WORKSPACE_VOLUME_NAME, + ] { + let template = SandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "volumes": [{ + "name": volume_name, + "persistent_volume_claim": {"claim_name": "pvc-user-data"} + }] + }))), + ..SandboxTemplate::default() + }; + + let err = KubernetesSandboxDriverConfig::from_template(&template).unwrap_err(); + assert!( + err.contains("reserved for OpenShell-managed volumes"), + "expected reserved volume name {volume_name:?} to be rejected, got {err}" + ); + } + } + + #[test] + fn reserved_kubernetes_volume_names_cover_managed_pod_volumes() { + let params = SandboxPodParams { + client_tls_secret_name: "openshell-client-tls-secret", + provider_spiffe_enabled: true, + provider_spiffe_workload_api_socket_path: "/spiffe-workload-api/spire-agent.sock", + ..SandboxPodParams::default() + }; + let pod_template = sandbox_template_to_k8s( + &SandboxTemplate::default(), + false, + &std::collections::HashMap::new(), + true, + ¶ms, + ); + let volume_names = pod_template["spec"]["volumes"] + .as_array() + .expect("volumes should exist") + .iter() + .filter_map(|volume| volume["name"].as_str()) + .collect::>(); + + for volume_name in volume_names { + assert!( + KUBERNETES_DRIVER_RESERVED_VOLUME_NAMES.contains(&volume_name), + "managed volume {volume_name:?} should be reserved" + ); + } + } + + #[test] + fn driver_config_rejects_runtime_provider_spiffe_mount_path() { + let spec = SandboxSpec { + template: Some(SandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "volumes": [{ + "name": "user-data", + "persistent_volume_claim": {"claim_name": "pvc-user-data"} + }], + "containers": { + "agent": { + "volume_mounts": [{ + "name": "user-data", + "mount_path": "/custom-spiffe" + }] + } + } + }))), + ..SandboxTemplate::default() + }), + ..SandboxSpec::default() + }; + + let err = + kubernetes_driver_config_for_spec(Some(&spec), Some("/custom-spiffe/spire-agent.sock")) + .unwrap_err(); + + assert!(err.contains("/custom-spiffe")); + } + #[test] fn validate_rejects_zero_gpu_count() { let sandbox = Sandbox { @@ -3942,7 +4828,7 @@ mod tests { .expect("volumes should exist"); let tls_vol = volumes .iter() - .find(|v| v["name"] == "openshell-client-tls") + .find(|v| v["name"] == CLIENT_TLS_VOLUME_NAME) .expect("TLS volume should exist"); assert_eq!( tls_vol["secret"]["defaultMode"], @@ -4461,7 +5347,7 @@ mod tests { && volume["csi"]["driver"] == "csi.spiffe.io" })); assert!(volumes.iter().any(|volume| { - volume["name"] == "openshell-sa-token" + volume["name"] == SERVICE_ACCOUNT_TOKEN_VOLUME_NAME && volume["projected"]["sources"][0]["serviceAccountToken"]["path"] == "token" })); @@ -4514,7 +5400,7 @@ mod tests { log_level: "debug".to_string(), ..SandboxSpec::default() }; - let cr = sandbox_to_k8s_spec(Some(&spec), &SandboxPodParams::default()); + let cr = sandbox_to_k8s_spec_for_test(Some(&spec), &SandboxPodParams::default()); let env = cr["spec"]["podTemplate"]["spec"]["containers"][0]["env"] .as_array() .unwrap(); @@ -4541,7 +5427,7 @@ mod tests { )]), ..SandboxSpec::default() }; - let cr = sandbox_to_k8s_spec(Some(&spec), &SandboxPodParams::default()); + let cr = sandbox_to_k8s_spec_for_test(Some(&spec), &SandboxPodParams::default()); let env = cr["spec"]["podTemplate"]["spec"]["containers"][0]["env"] .as_array() .unwrap(); diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 1401f9e7c6..e4a0b79189 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -2002,6 +2002,40 @@ mod tests { })); } + #[test] + fn driver_config_rejects_duplicate_mount_targets() { + use openshell_core::proto::compute::v1::{DriverSandboxSpec, DriverSandboxTemplate}; + + let mut sandbox = test_sandbox("test-id", "test-name"); + sandbox.spec = Some(DriverSandboxSpec { + template: Some(DriverSandboxTemplate { + driver_config: Some(json_struct(serde_json::json!({ + "mounts": [ + { + "type": "volume", + "source": "work-nfs", + "target": "/sandbox/work" + }, + { + "type": "tmpfs", + "target": "/sandbox/work" + } + ] + }))), + ..Default::default() + }), + ..Default::default() + }); + let config = test_config(); + + let err = try_build_container_spec_with_token(&sandbox, &config, None).unwrap_err(); + + assert!( + err.to_string() + .contains("duplicate podman driver_config mount target") + ); + } + #[test] fn driver_config_rejects_bind_mounts_unless_enabled() { use openshell_core::proto::compute::v1::{DriverSandboxSpec, DriverSandboxTemplate}; diff --git a/docs/reference/sandbox-compute-drivers.mdx b/docs/reference/sandbox-compute-drivers.mdx index dc8bdb2345..03a9b564f0 100644 --- a/docs/reference/sandbox-compute-drivers.mdx +++ b/docs/reference/sandbox-compute-drivers.mdx @@ -346,6 +346,69 @@ If Agent Sandbox is upgraded in place, restart the OpenShell gateway after the c `Sandbox.spec.volumeClaimTemplates` is immutable after creation. To change storage configuration, delete the sandbox and create a new one with the updated spec. +### Kubernetes Driver Config PVC Mounts + +Kubernetes driver config can mount existing PersistentVolumeClaims into the +agent container. Use this when storage is provisioned outside OpenShell and a +sandbox should mount selected PVC subpaths instead of using the default +OpenShell-created `/sandbox` workspace PVC. + +```shell +openshell sandbox create \ + --driver-config-json '{ + "kubernetes": { + "volumes": [{ + "name": "user-data", + "persistent_volume_claim": { + "claim_name": "pvc-user-data-123", + "read_only": false + } + }], + "containers": { + "agent": { + "volume_mounts": [ + { + "name": "user-data", + "mount_path": "/sandbox/.openshell/workspace", + "sub_path": "workspace", + "read_only": false + }, + { + "name": "user-data", + "mount_path": "/sandbox/.openshell/memory", + "sub_path": "memory", + "read_only": false + } + ] + } + } + } + }' \ + -- claude +``` + +Kubernetes PVC mount schema: + +| Field | Description | +|---|---| +| `volumes[].name` | Pod volume name. It must be a DNS-1123 label, unique, and not use OpenShell-managed volume names. | +| `volumes[].persistent_volume_claim.claim_name` | Existing PVC name in the sandbox namespace. It must be a DNS-1123 subdomain name. | +| `volumes[].persistent_volume_claim.read_only` | Optional. Defaults to `true`. Set `false` to allow read-write mounts. | +| `containers.agent.volume_mounts[].name` | References a volume declared in `volumes`. | +| `containers.agent.volume_mounts[].mount_path` | Absolute, normalized container path for the agent mount. | +| `containers.agent.volume_mounts[].sub_path` | Optional relative PVC subpath. Absolute paths and `..` are rejected. | +| `containers.agent.volume_mounts[].read_only` | Optional. Defaults to `true`. It cannot be `false` when the PVC volume is read-only. | + +OpenShell rejects duplicate volume names, mounts that reference unknown volumes, +protected mount targets, and mounts that replace OpenShell TLS, supervisor, +ServiceAccount token, or SPIFFE paths. Read-write PVC access requires +`read_only: false` on both the PVC volume and each writable mount. + +Any driver-config mount under `/sandbox` disables the default `/sandbox` +workspace PVC injection for that sandbox. Only the explicit mount paths persist +through the external PVC; other `/sandbox` paths come from the current sandbox +image. + ## Sandbox User Identity OpenShell accepts both the hardcoded username `"sandbox"` and numeric UIDs in `[1000, 2_000_000_000]` for the supervisor's process identity (the policy's `run_as_user` field). The driver resolves the UID at sandbox creation time and passes it to the supervisor via environment variables. From 40194f935ef6e29cb07500b9109314778ab6915c Mon Sep 17 00:00:00 2001 From: Tony Luo Date: Sun, 12 Jul 2026 00:40:36 +0800 Subject: [PATCH 10/13] fix(network): fail closed when credential placeholders cannot be rewritten (#2162) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(network): fail closed when credential placeholders cannot be rewritten When the credential rewriter degrades internally, the proxy forwarded the literal `openshell:resolve:env:` placeholder (or its provider alias marker) to the upstream instead of the resolved secret, leaking the reserved token on the wire and causing upstream auth failures (#2161). Two fail-open paths are closed: - secrets: `rewrite_http_header_block` returned the header block verbatim when no `SecretResolver` was available, so the fail-closed marker scan (which ran only on the resolved path) never saw the placeholder. It now scans the header region for reserved markers even with no resolver and returns `UnresolvedPlaceholderError` when one is present. Marker-free traffic still passes through unchanged. - proxy: when TLS was detected on a CONNECT but `tls_state` was `None` (ephemeral CA generation or CA file write failed at startup), the handler fell back to a raw `copy_bidirectional` tunnel, bypassing credential rewrite. Inside the proxy handler `tls_state` is `None` only on CA-init failure (`mode != Proxy` never starts the handler, and `tls: skip` is handled earlier), so it now refuses the connection with a 503 and a High-severity denial event instead of tunneling. The two startup CA-failure logs are raised from Medium to High. Tests: resolver=None with a placeholder in the request line, a header value, and the provider-alias form now fail closed; marker-free passthrough is unchanged; the relay integration test asserts the request is rejected before any byte reaches upstream; and the 503 fail-closed response contract is locked. Signed-off-by: Tony Luo * fix(network): refuse CONNECT before 200 when TLS termination is unavailable The fail-closed refusal for a terminating CONNECT with no TLS termination state (ephemeral CA init failed) was written after the 200 Connection Established response. Because a CONNECT client only sends its TLS ClientHello after reading the 200, the peek-based TLS detection is inherently post-200, so the 503 landed inside the established tunnel and surfaced to the client as a TLS protocol error rather than a readable status. An 'allowed CONNECT' event was also logged first. Move the decision to a pre-200 gate: query_tls_mode resolves purely from the policy decision + host/port (no peeked bytes), so the route's TLS treatment is known before the tunnel is acknowledged. When TLS state is absent and the route is not tls: skip, write the 503 as the first bytes on the socket, emit the High-severity Denied event, and close. tls: skip routes tunnel raw exactly as before, and no allowed-CONNECT event is emitted on the refusal path. The now-unreachable post-200 branch is kept as defense in depth but no longer writes an in-tunnel 503; it fails closed by dropping the connection instead. Add connection-level regression tests over a real loopback socket: the gate refuses with HTTP/1.1 503 as the first bytes for a terminating route, and writes nothing when TLS termination is present or the route is tls: skip. Signed-off-by: Tony Luo * fix(network): order the CONNECT TLS-unavailable refusal after SSRF Addresses the gator re-check on #2162. Ordering: the pre-200 fail-closed refusal ran before SSRF/allowed_ips validation, so during CA-init failure an internal-address CONNECT got a 503 tls_termination_unavailable instead of the normal 403 ssrf_denied, weakening operator visibility in degraded state. The SSRF branches now return validated addresses; the refusal runs after that validation (an internal address has already been denied with 403) but still before the upstream connect and before 200 Connection Established. effective_tls_skip is still resolved up front since the refusal consumes it. Tests: add connection-level regressions through the real handle_tcp_connection, driving a CONNECT from a child /bin/bash copy so the /proc process-identity binding resolves it against a permissive policy (the hot-swap test's identity pattern). They assert the first bytes are HTTP/1.1 503 for a terminating route with no TLS state, a 403 (not 503) for an internal address, and no refusal for a tls: skip route. These are gated to Linux at runtime (evaluate_opa_tcp needs /proc); a companion test verifies the OPA policy shape (glob allow, tls mode) on every platform so the precondition is locked where /proc is unavailable. rest.rs: tighten the fail-closed relay test to assert the forwarded buffer is_empty() rather than merely lacking the placeholder/secret. Signed-off-by: Tony Luo * test(network): keep the CONNECT handler test client fork-free The handler regression tests forked cat to read the proxy reply, so the client socket fd was inherited by a second process with a different binary. The identity resolver correctly denies that as ambiguous shared-socket ownership (the same invariant resolve_process_identity_denies_fork_exec_shared_socket_ambiguity pins), so the tests exercised the deny path instead of the allow path — and on busy CI runners the deny-path /proc fallback scan exceeded the test budget and looked like a hang. The client script now uses only bash builtins (exec, printf, read -d '') so exactly one process owns the socket, and the child is left to exit on EOF instead of being killed mid-read. Signed-off-by: Tony Luo * test(network): drive the CONNECT handler tests with an in-process client The child-process client (even fork-free) made the handler tests environment-sensitive: on CI runners with a busy or restricted /proc, resolving the child's socket ownership degraded into the whole-/proc fallback scan and a deny, which surfaced as a hang. The client is now an in-process TcpStream and the test policy allows current_exe(), so identity resolution binds the socket to the test process itself in the descendant scan — the same in-process pattern the passing resolve_process_identity tests rely on. The tls: skip test additionally asserts that the handler emitted no DenialEvent at any stage, so it can no longer pass vacuously on a policy or identity deny. Refusal budgets widened to 30s as a belt for slow runners; the refusals themselves return in milliseconds. Signed-off-by: Tony Luo * test(core): pin the percent-encoded marker no-resolver fail-closed path The no-resolver scan already catches the percent-encoded canonical marker through its decoded pass; this regression pins it: a request line carrying openshell%3Aresolve%3Aenv%3AKEY with no resolver must fail closed with UnresolvedPlaceholderError { location: header }. Signed-off-by: Tony Luo --------- Signed-off-by: Tony Luo --- crates/openshell-core/src/secrets.rs | 55 +- .../src/l7/rest.rs | 67 +-- .../openshell-supervisor-network/src/proxy.rs | 541 ++++++++++++++++-- .../openshell-supervisor-network/src/run.rs | 12 +- 4 files changed, 597 insertions(+), 78 deletions(-) diff --git a/crates/openshell-core/src/secrets.rs b/crates/openshell-core/src/secrets.rs index d71a2139c6..e93bdc5390 100644 --- a/crates/openshell-core/src/secrets.rs +++ b/crates/openshell-core/src/secrets.rs @@ -929,6 +929,20 @@ pub fn rewrite_http_header_block( resolver: Option<&SecretResolver>, ) -> Result { let Some(resolver) = resolver else { + // Fail-closed: with no resolver there is nothing that can rewrite a + // credential placeholder, so forwarding the block verbatim would leak + // the reserved token to the upstream. Scan the header region (mirroring + // the resolved-path scan below) and reject if any reserved marker is + // present. Marker-free traffic still passes through unchanged, which is + // the intended behavior when the endpoint injects no credentials. + let scan_end = raw + .windows(4) + .position(|w| w == b"\r\n\r\n") + .map_or(raw.len(), |p| raw.len().min(p + 4 + 256)); + let header_region = String::from_utf8_lossy(&raw[..scan_end]); + if contains_reserved_credential_marker(&header_region) { + return Err(UnresolvedPlaceholderError { location: "header" }); + } return Ok(RewriteResult { rewritten: raw.to_vec(), redacted_target: None, @@ -1889,11 +1903,44 @@ mod tests { } #[test] - fn no_resolver_passes_through_without_scanning() { - // Even if placeholders are present, None resolver means no scanning + fn no_resolver_with_placeholder_in_request_line_fails_closed() { + // Fail-closed: a placeholder in the request line with no resolver must + // never be forwarded verbatim — it would leak the reserved token. let raw = b"GET /api/openshell:resolve:env:KEY HTTP/1.1\r\nHost: x\r\n\r\n"; - let result = rewrite_http_header_block(raw, None).expect("should succeed"); - assert_eq!(raw.as_slice(), result.rewritten.as_slice()); + let err = rewrite_http_header_block(raw, None) + .expect_err("placeholder without resolver must fail closed"); + assert_eq!(err.location, "header"); + } + + #[test] + fn no_resolver_with_placeholder_in_header_value_fails_closed() { + let raw = + b"GET / HTTP/1.1\r\nAuthorization: Bearer openshell:resolve:env:KEY\r\nHost: x\r\n\r\n"; + let err = rewrite_http_header_block(raw, None) + .expect_err("placeholder without resolver must fail closed"); + assert_eq!(err.location, "header"); + } + + #[test] + fn no_resolver_with_percent_encoded_marker_fails_closed() { + // Percent-encoded form of the canonical marker: a client (or an evasive + // sender) can URL-encode the reserved token, and the no-resolver scan + // must still catch it via the percent-decoded pass. + let raw = b"GET /api/openshell%3Aresolve%3Aenv%3AKEY HTTP/1.1\r\nHost: x\r\n\r\n"; + let err = rewrite_http_header_block(raw, None) + .expect_err("percent-encoded placeholder without resolver must fail closed"); + assert_eq!(err.location, "header"); + } + + #[test] + fn no_resolver_with_provider_alias_marker_fails_closed() { + // The provider-shaped alias marker is also a reserved credential token + // and must fail closed when no resolver can rewrite it. + let raw = + b"GET / HTTP/1.1\r\nX-Api-Key: OPENSHELL-RESOLVE-ENV-CUSTOM_TOKEN\r\nHost: x\r\n\r\n"; + let err = rewrite_http_header_block(raw, None) + .expect_err("alias marker without resolver must fail closed"); + assert_eq!(err.location, "header"); } #[test] diff --git a/crates/openshell-supervisor-network/src/l7/rest.rs b/crates/openshell-supervisor-network/src/l7/rest.rs index 0558a67e55..5c62f6f9ee 100644 --- a/crates/openshell-supervisor-network/src/l7/rest.rs +++ b/crates/openshell-supervisor-network/src/l7/rest.rs @@ -4857,12 +4857,15 @@ mod tests { assert!(forwarded.contains("Host: integrate.api.nvidia.com\r\n")); } - /// Verifies that without a `SecretResolver` (i.e. the L4-only raw tunnel - /// path, or no TLS termination), credential placeholders pass through - /// unmodified. This documents the behavior that causes 401 errors when - /// `tls: terminate` is missing from the endpoint config. + /// Verifies that when a request carries a credential placeholder but no + /// `SecretResolver` is available to rewrite it, the relay fails closed + /// instead of forwarding the reserved token verbatim. A `None` resolver + /// here stands in for a degraded internal state (e.g. secrets could not be + /// wired up); forwarding the placeholder would leak it to the upstream and + /// cause 401s. The fail-closed scan in `rewrite_http_header_block` rejects + /// the request before any byte reaches upstream. #[tokio::test] - async fn relay_request_without_resolver_leaks_placeholders() { + async fn relay_request_without_resolver_fails_closed_on_placeholder() { let (child_env, _resolver) = SecretResolver::from_provider_env( [("NVIDIA_API_KEY".to_string(), "nvapi-secret".to_string())] .into_iter() @@ -4887,55 +4890,39 @@ mod tests { body_length: BodyLength::ContentLength(2), }; - let upstream_task = tokio::spawn(async move { - let mut buf = vec![0u8; 4096]; - let mut total = 0; - loop { - let n = upstream_side.read(&mut buf[total..]).await.unwrap(); - if n == 0 { - break; - } - total += n; - if let Some(hdr_end) = buf[..total].windows(4).position(|w| w == b"\r\n\r\n") { - if total >= hdr_end + 4 + 2 { - break; - } - } - } - upstream_side - .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") - .await - .unwrap(); - upstream_side.flush().await.unwrap(); - String::from_utf8_lossy(&buf[..total]).to_string() - }); - - // Pass `None` for the resolver — simulates the L4 path where no - // rewriting occurs. + // Pass `None` for the resolver — simulates a degraded state where no + // rewriting can occur. The relay must reject the request. let relay = tokio::time::timeout( std::time::Duration::from_secs(5), relay_http_request_with_resolver( &req, &mut proxy_to_client, &mut proxy_to_upstream, - None, // <-- No resolver, as in the L4 raw tunnel path + None, ), ) .await .expect("relay must not deadlock"); - relay.expect("relay should succeed"); - - let forwarded = upstream_task.await.expect("upstream task should complete"); - // Without a resolver, the placeholder LEAKS to upstream — this is the - // documented behavior that causes 401s when `tls: terminate` is missing. assert!( - forwarded.contains("openshell:resolve:env:NVIDIA_API_KEY"), - "Expected placeholder to leak without resolver, got: {forwarded}" + relay.is_err(), + "relay must fail closed when a placeholder cannot be resolved" ); + + // The fail-closed scan rejects the request before any byte reaches + // upstream, so nothing at all must have been forwarded — a strictly + // stronger invariant than "the placeholder/secret did not appear". Drop + // the proxy write half so the read observes EOF instead of blocking. + drop(proxy_to_upstream); + let mut forwarded = Vec::new(); + upstream_side + .read_to_end(&mut forwarded) + .await + .expect("upstream read should complete"); assert!( - !forwarded.contains("nvapi-secret"), - "Real secret should NOT appear without resolver, got: {forwarded}" + forwarded.is_empty(), + "no bytes may reach upstream when the placeholder cannot be resolved; got: {}", + String::from_utf8_lossy(&forwarded) ); } diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index c0c33e6fd1..38cab79c79 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -743,6 +743,15 @@ async fn handle_tcp_connection( return Ok(()); } + // Resolve the route's TLS treatment up front. `query_tls_mode` reads only + // the policy decision + host/port (no peeked bytes), so it is valid before + // the `200`. The fail-closed refusal that consumes it runs after the SSRF/ + // allowed_ips validation below — so an internal-address CONNECT still gets + // the SSRF 403 and telemetry in degraded state — but before the upstream + // connect and before `200 Connection Established`. + let effective_tls_skip = + query_tls_mode(&opa_engine, &decision, &host_lc, port) == crate::l7::TlsMode::Skip; + let sandbox_entrypoint_pid = entrypoint_pid.load(Ordering::Acquire); // Query allowed_ips from the matched endpoint config (if any). @@ -764,7 +773,7 @@ async fn handle_tcp_connection( // The "non-empty" branch is the explicit-allowlist path; reading it first // matches the policy decision narrative. #[allow(clippy::if_not_else)] - let mut upstream = if is_host_gateway_alias(&host_lc) + let validated_addrs = if is_host_gateway_alias(&host_lc) && let Some(gw) = *trusted_host_gateway { // Trusted host-gateway path. The compute driver injected this hostname @@ -773,9 +782,7 @@ async fn handle_tcp_connection( // addresses (used by rootless Podman with pasta) are not hard-blocked. // Cloud metadata IPs and control-plane ports are still rejected. match resolve_and_check_trusted_gateway(&host, port, gw, sandbox_entrypoint_pid).await { - Ok(addrs) => TcpStream::connect(addrs.as_slice()) - .await - .into_diagnostic()?, + Ok(addrs) => addrs, Err(reason) => { { let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) @@ -829,9 +836,7 @@ async fn handle_tcp_connection( match resolve_and_check_allowed_ips(&host, port, &nets, sandbox_entrypoint_pid) .await { - Ok(addrs) => TcpStream::connect(addrs.as_slice()) - .await - .into_diagnostic()?, + Ok(addrs) => addrs, Err(reason) => { { let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) @@ -931,9 +936,7 @@ async fn handle_tcp_connection( // the resolved IP in allowed_ips. Always-blocked addresses and // control-plane ports remain denied. match resolve_and_check_declared_endpoint(&host, port, sandbox_entrypoint_pid).await { - Ok(addrs) => TcpStream::connect(addrs.as_slice()) - .await - .into_diagnostic()?, + Ok(addrs) => addrs, Err(reason) => { { let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) @@ -983,9 +986,7 @@ async fn handle_tcp_connection( } else { // Default: reject all internal IPs (loopback, RFC 1918, link-local). match resolve_and_reject_internal(&host, port, sandbox_entrypoint_pid).await { - Ok(addrs) => TcpStream::connect(addrs.as_slice()) - .await - .into_diagnostic()?, + Ok(addrs) => addrs, Err(reason) => { { let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) @@ -1033,6 +1034,52 @@ async fn handle_tcp_connection( } }; + // Fail closed AFTER SSRF/allowed_ips validation (an internal-address CONNECT + // has already returned the SSRF 403 above) but BEFORE connecting upstream + // and BEFORE `200 Connection Established`. A terminating route with no TLS + // termination state cannot rewrite credential placeholders; the 503 must be + // the first bytes on the socket rather than a post-200 in-tunnel write the + // client would misread as a TLS error. No "allowed CONNECT" event is emitted + // for this path — that log lives after the `200` below. + if refuse_connect_when_tls_unavailable(&mut client, tls_state.is_some(), effective_tls_skip) + .await? + { + let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Open) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::High) + .status(StatusId::Failure) + .dst_endpoint(Endpoint::from_domain(&host_lc, port)) + .src_endpoint_addr(peer_addr.ip(), peer_addr.port()) + .actor_process( + Process::from_bypass(&binary_str, &pid_str, &ancestors_str) + .with_cmd_line(&cmdline_str), + ) + .firewall_rule(policy_str, "tls") + .message(format!( + "CONNECT refused for {host_lc}:{port}: {TLS_TERMINATION_UNAVAILABLE_DETAIL}" + )) + .status_detail(TLS_TERMINATION_UNAVAILABLE_DETAIL) + .build(); + ocsf_emit!(event); + emit_activity_simple(activity_tx.as_ref(), true, "tls_termination_unavailable"); + emit_denial( + &denial_tx, + &host_lc, + port, + &binary_str, + &decision, + TLS_TERMINATION_UNAVAILABLE_DETAIL, + "connect-tls-termination-unavailable", + ); + return Ok(()); + } + + let mut upstream = TcpStream::connect(validated_addrs.as_slice()) + .await + .into_diagnostic()?; + debug!( "handle_tcp_connection dns_resolve_and_tcp_connect: {}ms host={host_lc}", dns_connect_start.elapsed().as_millis() @@ -1072,10 +1119,8 @@ async fn handle_tcp_connection( } emit_connect_activity_if_l4_only(&activity_tx, l7_route.as_ref()); - // Determine effective TLS mode. Check the raw endpoint config for - // `tls: skip` independently of L7 config (which requires `protocol`). - let effective_tls_skip = - query_tls_mode(&opa_engine, &decision, &host_lc, port) == crate::l7::TlsMode::Skip; + // `effective_tls_skip` was resolved before the `200` above (the fail-closed + // gate needs it) and drives the raw-tunnel branch below. // Build L7 eval context (shared by TLS-terminated and plaintext paths). let ctx = crate::l7::relay::L7EvalContext { @@ -1209,21 +1254,48 @@ async fn handle_tcp_connection( } } } else { - { - let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::Low) - .status(StatusId::Failure) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .message(format!( - "TLS detected but TLS state not configured for {host_lc}:{port}, falling back to raw tunnel" - )) - .build(); - ocsf_emit!(event); - } - let _ = tokio::io::copy_bidirectional(&mut client, &mut upstream) - .await - .into_diagnostic()?; + // Defense in depth; unreachable in normal operation. The pre-200 + // fail-closed gate already refuses terminating routes when no TLS + // termination state exists, and `tls: skip` routes raw-tunnel + // before this peek. Reaching here means a future refactor bypassed + // that gate. The `200 Connection Established` was already sent, so + // the tunnel is live: any HTTP bytes now would be decoded as a TLS + // protocol error, so we fail closed by DROPPING the connection + // rather than raw-tunneling (which would forward the client's TLS + // stream upstream and leak any `openshell:resolve:env:*` + // placeholder verbatim). + const DETAIL: &str = "TLS termination unavailable after tunnel establishment; \ + closing connection — credential rewrite would be bypassed"; + let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Open) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::High) + .status(StatusId::Failure) + .dst_endpoint(Endpoint::from_domain(&host_lc, port)) + .src_endpoint_addr(peer_addr.ip(), peer_addr.port()) + .actor_process( + Process::from_bypass(&binary_str, &pid_str, &ancestors_str) + .with_cmd_line(&cmdline_str), + ) + .firewall_rule(policy_str, "tls") + .message(format!("CONNECT refused for {host_lc}:{port}: {DETAIL}")) + .status_detail(DETAIL) + .build(); + ocsf_emit!(event); + emit_activity_simple(activity_tx.as_ref(), true, "tls_termination_unavailable"); + emit_denial( + &denial_tx, + &host_lc, + port, + &binary_str, + &decision, + DETAIL, + "connect-tls-termination-unavailable", + ); + // No HTTP response: the tunnel is already established, so writing + // bytes here would corrupt the client's TLS handshake. Drop instead. + return Ok(()); } } else if tunnel_protocol == TunnelProtocol::Http1 { // Plaintext HTTP detected. @@ -4499,6 +4571,50 @@ fn build_json_error_response(status: u16, status_text: &str, error: &str, detail .into_bytes() } +/// Detail shared by the fail-closed 503 body, the OCSF denial event, and the +/// denial notification when a terminating CONNECT route has no TLS termination +/// state available. +const TLS_TERMINATION_UNAVAILABLE_DETAIL: &str = "TLS termination unavailable (CA initialization failed); \ + refusing to tunnel — credential rewrite would be bypassed"; + +/// Fail-closed gate evaluated BEFORE `200 Connection Established`. +/// +/// A CONNECT route that terminates TLS (`tls: skip` is exempt) relies on the +/// ephemeral CA to rewrite credential placeholders. When no TLS termination +/// state exists — the CA failed to generate or write at startup (`run.rs`); +/// `mode != Proxy` never starts this handler — the proxy cannot rewrite those +/// placeholders, so raw-tunneling would forward the client's TLS stream +/// straight upstream and leak any `openshell:resolve:env:*` placeholder +/// verbatim. +/// +/// The refusal is written here, as the first bytes on the socket, because a +/// CONNECT client only sends its TLS `ClientHello` after reading the `200`. +/// Refusing after the `200` would land the 503 inside the established tunnel, +/// where the client decodes it as a TLS protocol error rather than a readable +/// HTTP status (the flaw this replaces). Returns `true` when the connection was +/// refused (the caller must stop) and `false` when the caller should proceed to +/// establish the tunnel. +async fn refuse_connect_when_tls_unavailable( + client: &mut TcpStream, + tls_state_present: bool, + effective_tls_skip: bool, +) -> Result { + if tls_state_present || effective_tls_skip { + return Ok(false); + } + respond( + client, + &build_json_error_response( + 503, + "Service Unavailable", + "tls_termination_unavailable", + TLS_TERMINATION_UNAVAILABLE_DETAIL, + ), + ) + .await?; + Ok(true) +} + /// Check if a miette error represents a benign connection close. /// /// TLS handshake EOF, missing `close_notify`, connection resets, and broken @@ -8167,6 +8283,367 @@ network_policies: assert_eq!(body["detail"], "connection to api.example.com:443 failed"); } + /// Locks the fail-closed response the CONNECT handler sends when TLS is + /// detected but no termination state exists (ephemeral CA setup failed). + /// The proxy must refuse the connection with a 503 instead of raw-tunneling + /// the TLS stream, which would bypass credential rewrite and leak + /// placeholders verbatim. + #[test] + fn test_json_error_response_503_tls_termination_unavailable() { + let detail = TLS_TERMINATION_UNAVAILABLE_DETAIL; + let resp = build_json_error_response( + 503, + "Service Unavailable", + "tls_termination_unavailable", + detail, + ); + let resp_str = String::from_utf8(resp).unwrap(); + + assert!(resp_str.starts_with("HTTP/1.1 503 Service Unavailable\r\n")); + assert!(resp_str.contains("Connection: close\r\n")); + + let body_start = resp_str.find("\r\n\r\n").unwrap() + 4; + let body: serde_json::Value = serde_json::from_str(&resp_str[body_start..]).unwrap(); + assert_eq!(body["error"], "tls_termination_unavailable"); + assert_eq!(body["detail"], detail); + } + + /// Connection-level regression for the pre-200 fail-closed gate. A real + /// loopback client opens a connection and the proxy-side gate runs with no + /// TLS termination state for a terminating (non-`tls: skip`) route. The + /// FIRST bytes the client reads back must be a real `HTTP/1.1 503`, not the + /// `HTTP/1.1 200 Connection Established` that would establish the tunnel and + /// bury the refusal inside it as a TLS protocol error (PR #2162 gator flaw). + #[tokio::test] + async fn connect_without_tls_termination_refused_with_503_before_200() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let mut client = TcpStream::connect(addr).await.unwrap(); + let (mut server, _) = listener.accept().await.unwrap(); + + // Terminating route (tls: skip = false) with no termination state. + let refused = refuse_connect_when_tls_unavailable(&mut server, false, false) + .await + .expect("gate write should succeed"); + assert!( + refused, + "gate must refuse when TLS termination is unavailable" + ); + + let mut buf = vec![0u8; 512]; + let n = client.read(&mut buf).await.unwrap(); + let response = String::from_utf8_lossy(&buf[..n]); + assert!( + response.starts_with("HTTP/1.1 503 Service Unavailable\r\n"), + "client must read a 503 as the first bytes, not a 200; got: {response}" + ); + assert!( + !response.contains("200 Connection Established"), + "no tunnel must be established before the refusal; got: {response}" + ); + + let body_start = response.find("\r\n\r\n").unwrap() + 4; + let body: serde_json::Value = serde_json::from_str(&response[body_start..]).unwrap(); + assert_eq!(body["error"], "tls_termination_unavailable"); + assert_eq!(body["detail"], TLS_TERMINATION_UNAVAILABLE_DETAIL); + } + + /// The pre-200 gate must NOT touch the socket when the route can proceed: + /// either TLS termination state is present, or the route is `tls: skip` + /// (raw tunnel, no termination or credential injection). In both cases the + /// gate returns `false` and writes nothing, letting the caller send its + /// own `200 Connection Established`. + #[tokio::test] + async fn connect_with_tls_termination_or_skip_proceeds_without_writing() { + for (tls_present, tls_skip) in [(true, false), (false, true), (true, true)] { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let mut client = TcpStream::connect(addr).await.unwrap(); + let (mut server, _) = listener.accept().await.unwrap(); + + let refused = refuse_connect_when_tls_unavailable(&mut server, tls_present, tls_skip) + .await + .expect("gate should succeed"); + assert!( + !refused, + "gate must proceed (tls_present={tls_present}, tls_skip={tls_skip})" + ); + + // Nothing was written; after dropping the server the client sees a + // clean EOF (0 bytes) rather than any buffered response. + drop(server); + let mut buf = vec![0u8; 16]; + let n = client.read(&mut buf).await.unwrap(); + assert_eq!( + n, 0, + "gate must not write to the socket when proceeding \ + (tls_present={tls_present}, tls_skip={tls_skip})" + ); + } + } + + /// Drives a real `CONNECT` through the full `handle_tcp_connection` entry + /// point and returns `(completed, client_response_bytes, denial_stages)`. + /// `completed` is `false` when the handler was still running at `budget` + /// (e.g. a `tls: skip` route that proceeded past the fail-closed gate and + /// blocked on the unroutable upstream connect). `denial_stages` collects the + /// `denial_stage` of every `DenialEvent` the handler emitted — empty means + /// the connection was allowed and not blocked/refused at any stage. + /// + /// The client is an in-process `TcpStream` (no child process), so the client + /// socket is owned solely by this test process. Identity resolution finds + /// that owner in the descendant scan (which includes the entrypoint PID + /// itself), binds to `current_exe()`, and never falls through to the + /// whole-`/proc` scan — the environment-sensitive path that made a forked + /// child flaky under a busy CI `/proc`. Callers gate on Linux; + /// `evaluate_opa_tcp` denies unconditionally without `/proc`. + async fn drive_connect_through_handler( + endpoint_yaml: &str, + connect_target: &str, + budget: std::time::Duration, + ) -> (bool, Vec, Vec) { + const POLICY_REGO: &str = include_str!("../data/sandbox-policy.rego"); + + // Allow the current test binary — the in-process client's + // `/proc//exe` — to reach the endpoint. Matching is by path. + let exe = std::env::current_exe().expect("current_exe"); + let data = format!( + r#"network_policies: + test_allow: + name: test_allow + endpoints: +{endpoint_yaml} binaries: + - {{ path: "{exe}" }} +"#, + exe = exe.display(), + ); + let engine = Arc::new(OpaEngine::from_strings(POLICY_REGO, &data).expect("load policy")); + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let proxy_port = listener.local_addr().unwrap().port(); + + // In-process client: connect, send the CONNECT request, read the reply to + // EOF. The proxy closes the socket after writing a 503/403; a proceeding + // `tls: skip` route stalls on the upstream connect until the handler is + // dropped at `budget`, which closes the socket and ends the read. + let target = connect_target.to_string(); + let client = tokio::spawn(async move { + let mut sock = TcpStream::connect(("127.0.0.1", proxy_port)).await.unwrap(); + let req = format!("CONNECT {target} HTTP/1.1\r\nHost: {target}\r\n\r\n"); + sock.write_all(req.as_bytes()).await.unwrap(); + let mut buf = Vec::new(); + let _ = sock.read_to_end(&mut buf).await; + buf + }); + + let (server, _peer) = listener.accept().await.unwrap(); + let entrypoint_pid = Arc::new(AtomicU32::new(std::process::id())); + let cache = Arc::new(BinaryIdentityCache::new()); + let (denial_tx, mut denial_rx) = mpsc::unbounded_channel(); + + let completed = tokio::time::timeout( + budget, + Box::pin(handle_tcp_connection( + server, + engine, + cache, + entrypoint_pid, + None, // tls_state — ephemeral CA unavailable + None, // inference_ctx + None, // policy_local_ctx + Arc::new(None), // trusted_host_gateway + None, // secret_resolver + None, // dynamic_credentials + Some(denial_tx), // denial_tx — positive allow/deny signal + None, // activity_tx + )), + ) + .await + .is_ok(); + + let stdout = client.await.expect("client task"); + + // The handler future is now finished or dropped, so its sender half is + // gone; drain every denial it emitted. + let mut denial_stages = Vec::new(); + while let Ok(event) = denial_rx.try_recv() { + denial_stages.push(event.denial_stage); + } + (completed, stdout, denial_stages) + } + + /// End-to-end regression for the gator finding on PR #2162: with no TLS + /// termination state, a terminating `CONNECT` must have its 503 written as + /// the FIRST bytes on the socket — never after a `200 Connection + /// Established` that would bury the refusal inside the tunnel as a TLS error. + #[tokio::test] + async fn connect_handler_refuses_terminating_route_with_503_before_200() { + if !cfg!(target_os = "linux") { + eprintln!("skipping: handler identity binding requires /proc (Linux)"); + return; + } + + // Public documentation IP (RFC 5737 TEST-NET-3): passes the SSRF check + // but is never actually connected — the fail-closed gate refuses first. + // The 30s budget is a generous belt against a slow CI runner; the refusal + // returns in milliseconds. + let (completed, stdout, _denials) = drive_connect_through_handler( + " - { host: \"203.0.113.10\", port: 443 }\n", + "203.0.113.10:443", + std::time::Duration::from_secs(30), + ) + .await; + + assert!(completed, "the refusal must return promptly, not hang"); + let resp = String::from_utf8_lossy(&stdout); + assert!( + resp.starts_with("HTTP/1.1 503 Service Unavailable"), + "first bytes must be a 503, not a 200; got: {resp:?}" + ); + assert!( + !resp.contains("200 Connection Established"), + "no tunnel must be established before the refusal; got: {resp:?}" + ); + assert!( + resp.contains("tls_termination_unavailable"), + "must be the TLS-termination-unavailable refusal; got: {resp:?}" + ); + } + + /// Ordering regression (gator re-check item 1): during CA-init failure, an + /// internal-address `CONNECT` must still receive the SSRF `403`, not the + /// TLS-unavailable `503` — SSRF validation runs before the fail-closed gate. + #[tokio::test] + async fn connect_handler_returns_ssrf_403_for_internal_address_not_503() { + if !cfg!(target_os = "linux") { + eprintln!("skipping: handler identity binding requires /proc (Linux)"); + return; + } + + let (completed, stdout, _denials) = drive_connect_through_handler( + " - { host: \"127.0.0.1\", port: 443 }\n", + "127.0.0.1:443", + std::time::Duration::from_secs(30), + ) + .await; + + assert!(completed, "the SSRF denial must return promptly, not hang"); + let resp = String::from_utf8_lossy(&stdout); + assert!( + resp.starts_with("HTTP/1.1 403 Forbidden"), + "internal address must get the SSRF 403; got: {resp:?}" + ); + assert!( + resp.contains("ssrf_denied"), + "must be an SSRF denial; got: {resp:?}" + ); + assert!( + !resp.contains("503"), + "SSRF runs before the fail-closed gate, so the 503 must not appear; got: {resp:?}" + ); + } + + /// A real `tls: skip` policy path through the handler is exempt from the + /// fail-closed gate even with no TLS termination state: the handler proceeds + /// past the refusal to the raw-tunnel upstream connect (which stalls on the + /// unroutable target), emitting no 503 and no client response before it. + #[tokio::test] + async fn connect_handler_tls_skip_route_is_not_refused() { + if !cfg!(target_os = "linux") { + eprintln!("skipping: handler identity binding requires /proc (Linux)"); + return; + } + + // `_completed` is intentionally ignored: whether the unroutable connect + // stalls (times out) or is rejected fast by a CI egress firewall, the + // invariant is the same — a `tls: skip` route is exempt from the + // fail-closed gate, so it proceeds to the tunnel connect and writes no + // 503/denial to the client beforehand. + let (_completed, stdout, denial_stages) = drive_connect_through_handler( + " - { host: \"203.0.113.10\", port: 443, tls: skip }\n", + "203.0.113.10:443", + std::time::Duration::from_millis(800), + ) + .await; + + let resp = String::from_utf8_lossy(&stdout); + assert!( + stdout.is_empty(), + "tls: skip must emit no refusal/denial before the tunnel connect; got: {resp:?}" + ); + // Positive allow evidence, so this test cannot pass vacuously on a + // policy/identity deny (which would emit a DenialEvent): the handler + // must have emitted no denial at any stage. + assert!( + denial_stages.is_empty(), + "tls: skip must be allowed end-to-end, not denied at any stage; got: {denial_stages:?}" + ); + } + + /// Verifies the policy half of the Linux handler tests on every platform + /// (no `/proc` needed): the temp-dir binary glob yields an Allow for a + /// literal-IP endpoint, `tls: skip` reads back as `TlsMode::Skip`, and a + /// terminating endpoint reads back as `TlsMode::Auto`. Locks the OPA + /// preconditions so a policy-format regression is caught even where the + /// process-identity binding can't run. + #[test] + fn handler_test_policy_allows_glob_binary_and_reads_tls_mode() { + const POLICY_REGO: &str = include_str!("../data/sandbox-policy.rego"); + + let eval = |tls_line: &str| { + let data = format!( + r#"network_policies: + test_allow: + name: test_allow + endpoints: + - {{ host: "203.0.113.10", port: 443{tls_line} }} + binaries: + - {{ path: "/tmp/openshell-connect-test/*" }} +"# + ); + let engine = OpaEngine::from_strings(POLICY_REGO, &data).expect("load policy"); + let input = crate::opa::NetworkInput { + host: "203.0.113.10".to_string(), + port: 443, + binary_path: PathBuf::from("/tmp/openshell-connect-test/connect-bash"), + binary_sha256: "unused".to_string(), + ancestors: vec![], + cmdline_paths: vec![], + }; + let (action, generation) = engine + .evaluate_network_action_with_generation(&input) + .expect("evaluate"); + match &action { + NetworkAction::Allow { matched_policy } => { + assert!(matched_policy.is_some(), "allow must carry the policy name"); + } + NetworkAction::Deny { reason } => { + panic!("glob binary must be allowed, got deny: {reason}") + } + } + let decision = ConnectDecision { + action, + generation, + binary: Some(input.binary_path), + binary_pid: Some(1), + ancestors: vec![], + cmdline_paths: vec![], + }; + query_tls_mode(&engine, &decision, "203.0.113.10", 443) + }; + + assert_eq!( + eval(", tls: skip"), + crate::l7::TlsMode::Skip, + "tls: skip endpoint must resolve to TlsMode::Skip" + ); + assert_eq!( + eval(""), + crate::l7::TlsMode::Auto, + "terminating endpoint must resolve to TlsMode::Auto" + ); + } + #[test] fn test_json_error_response_content_length_matches() { let resp = build_json_error_response(403, "Forbidden", "test", "detail"); diff --git a/crates/openshell-supervisor-network/src/run.rs b/crates/openshell-supervisor-network/src/run.rs index 8e17758bd6..d68db6f1c3 100644 --- a/crates/openshell-supervisor-network/src/run.rs +++ b/crates/openshell-supervisor-network/src/run.rs @@ -225,9 +225,13 @@ pub async fn run_networking( (Some(state), Some(paths)) } Err(e) => { + // High severity: with TLS termination disabled the proxy + // cannot rewrite credentials, so it fails closed on + // TLS-bearing connections (see proxy.rs) rather than + // leaking placeholders through a raw tunnel. ocsf_emit!( ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) + .severity(SeverityId::High) .status(StatusId::Failure) .state(StateId::Disabled, "disabled") .message(format!( @@ -240,9 +244,13 @@ pub async fn run_networking( } } Err(e) => { + // High severity: with TLS termination disabled the proxy cannot + // rewrite credentials, so it fails closed on TLS-bearing + // connections (see proxy.rs) rather than leaking placeholders + // through a raw tunnel. ocsf_emit!( ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) + .severity(SeverityId::High) .status(StatusId::Failure) .state(StateId::Disabled, "disabled") .message(format!( From bb72d0123c748ed7e209880f7bab593e10aae221 Mon Sep 17 00:00:00 2001 From: Adel Zaalouk Date: Mon, 13 Jul 2026 02:28:57 +0200 Subject: [PATCH 11/13] fix(server): allow newlines in exec command arguments (#1965) --- crates/openshell-server/src/grpc/sandbox.rs | 73 ++++++++++++++++--- .../openshell-server/src/grpc/validation.rs | 70 +++++++++++++++++- 2 files changed, 130 insertions(+), 13 deletions(-) diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index 04d5a4ed52..542fee97c0 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -1466,9 +1466,6 @@ fn shell_escape(value: &str) -> Result { if value.bytes().any(|b| b == 0) { return Err("value contains null bytes".to_string()); } - if value.bytes().any(|b| b == b'\n' || b == b'\r') { - return Err("value contains newline or carriage return".to_string()); - } if value.is_empty() { return Ok("''".to_string()); } @@ -1555,7 +1552,11 @@ async fn stream_exec_over_relay( timeout_seconds: u32, request_tty: bool, ) -> Result<(), Status> { - let command_preview: String = command.chars().take(120).collect(); + let command_preview: String = command + .chars() + .take(120) + .flat_map(char::escape_default) + .collect(); info!( sandbox_id = %sandbox_id, channel_id = %channel_id, @@ -1631,7 +1632,11 @@ async fn stream_interactive_exec_over_relay( cols: u32, rows: u32, ) -> Result<(), Status> { - let command_preview: String = command.chars().take(120).collect(); + let command_preview: String = command + .chars() + .take(120) + .flat_map(char::escape_default) + .collect(); info!( sandbox_id = %sandbox_id, channel_id = %channel_id, @@ -2061,10 +2066,20 @@ mod tests { } #[test] - fn shell_escape_rejects_newlines() { - assert!(shell_escape("line1\nline2").is_err()); - assert!(shell_escape("line1\rline2").is_err()); - assert!(shell_escape("line1\r\nline2").is_err()); + fn shell_escape_allows_newlines() { + assert!(shell_escape("line1\nline2").is_ok()); + assert!(shell_escape("line1\rline2").is_ok()); + assert!(shell_escape("line1\r\nline2").is_ok()); + } + + #[test] + fn shell_escape_preserves_newlines_in_single_quotes() { + assert_eq!(shell_escape("line1\nline2").unwrap(), "'line1\nline2'"); + assert_eq!( + shell_escape("def f():\n return 1").unwrap(), + "'def f():\n return 1'" + ); + assert_eq!(shell_escape("line1\r\nline2").unwrap(), "'line1\r\nline2'"); } // ---- build_remote_exec_command ---- @@ -2120,7 +2135,45 @@ mod tests { workdir: "/tmp\nmalicious".to_string(), ..Default::default() }; - assert!(build_remote_exec_command(&req).is_err()); + // Validation layer rejects newlines in workdir + assert!(validate_exec_request_fields(&req).is_err()); + } + + #[test] + fn build_remote_exec_command_accepts_multiline_script() { + use openshell_core::proto::ExecSandboxRequest; + let req = ExecSandboxRequest { + sandbox_id: "test".to_string(), + command: vec![ + "python3".to_string(), + "-c".to_string(), + "def f():\n return 1\nprint(f())".to_string(), + ], + ..Default::default() + }; + let cmd = build_remote_exec_command(&req).unwrap(); + assert!(cmd.starts_with("python3 -c ")); + assert!(cmd.contains("'def f():\n return 1\nprint(f())'")); + } + + #[test] + fn build_remote_exec_command_multiline_with_single_quotes() { + use openshell_core::proto::ExecSandboxRequest; + let req = ExecSandboxRequest { + sandbox_id: "test".to_string(), + command: vec![ + "python3".to_string(), + "-c".to_string(), + "print('one')\r\nprint('two')".to_string(), + ], + ..Default::default() + }; + let cmd = build_remote_exec_command(&req).unwrap(); + assert!(cmd.starts_with("python3 -c ")); + assert!( + cmd.contains("'print('\"'\"'one'\"'\"')\r\nprint('\"'\"'two'\"'\"')'"), + "CR/LF with embedded single quotes must compose correctly: {cmd}" + ); } #[test] diff --git a/crates/openshell-server/src/grpc/validation.rs b/crates/openshell-server/src/grpc/validation.rs index 0b3548b062..d8c02e7500 100644 --- a/crates/openshell-server/src/grpc/validation.rs +++ b/crates/openshell-server/src/grpc/validation.rs @@ -32,8 +32,10 @@ pub(super) const MAX_EXEC_ARG_LEN: usize = 32 * 1024; // 32 KiB /// Maximum length of the workdir field (bytes). pub(super) const MAX_EXEC_WORKDIR_LEN: usize = 4096; -/// Validate fields of an `ExecSandboxRequest` for control characters and size -/// limits before constructing a shell command string. +/// Validate exec request size limits and field-specific character constraints. +/// +/// Command arguments only reject NUL (newlines are valid for inline scripts). +/// Environment values and workdir reject both NUL and newlines. pub(super) fn validate_exec_request_fields(req: &ExecSandboxRequest) -> Result<(), Status> { if req.command.len() > MAX_EXEC_COMMAND_ARGS { return Err(Status::invalid_argument(format!( @@ -46,7 +48,7 @@ pub(super) fn validate_exec_request_fields(req: &ExecSandboxRequest) -> Result<( "command argument {i} exceeds {MAX_EXEC_ARG_LEN} byte limit" ))); } - reject_control_chars(arg, &format!("command argument {i}"))?; + reject_null_char(arg, &format!("command argument {i}"))?; } for (key, value) in &req.environment { if value.len() > MAX_EXEC_ARG_LEN { @@ -70,11 +72,23 @@ pub(super) fn validate_exec_request_fields(req: &ExecSandboxRequest) -> Result<( /// Reject null bytes and newlines in a user-supplied value. pub(super) fn reject_control_chars(value: &str, field_name: &str) -> Result<(), Status> { + reject_null_char(value, field_name)?; + reject_newline_chars(value, field_name)?; + Ok(()) +} + +/// Reject null bytes in a user-supplied value. +pub(super) fn reject_null_char(value: &str, field_name: &str) -> Result<(), Status> { if value.bytes().any(|b| b == 0) { return Err(Status::invalid_argument(format!( "{field_name} contains null bytes" ))); } + Ok(()) +} + +/// Reject newline and carriage return characters in a user-supplied value. +pub(super) fn reject_newline_chars(value: &str, field_name: &str) -> Result<(), Status> { if value.bytes().any(|b| b == b'\n' || b == b'\r') { return Err(Status::invalid_argument(format!( "{field_name} contains newline or carriage return characters" @@ -1800,4 +1814,54 @@ mod tests { assert!(reject_control_chars("line1\nline2", "test").is_err()); assert!(reject_control_chars("line1\rline2", "test").is_err()); } + + #[test] + fn validate_exec_allows_newlines_in_command_args() { + let req = ExecSandboxRequest { + sandbox_id: "test".to_string(), + command: vec![ + "python3".to_string(), + "-c".to_string(), + "def f():\n return 1\nprint(f())".to_string(), + ], + ..Default::default() + }; + assert!(validate_exec_request_fields(&req).is_ok()); + } + + #[test] + fn validate_exec_still_rejects_null_bytes_in_command_args() { + let req = ExecSandboxRequest { + sandbox_id: "test".to_string(), + command: vec!["echo".to_string(), "hello\x00world".to_string()], + ..Default::default() + }; + let err = validate_exec_request_fields(&req).unwrap_err(); + assert!(err.message().contains("null")); + } + + #[test] + fn validate_exec_still_rejects_newlines_in_workdir() { + let req = ExecSandboxRequest { + sandbox_id: "test".to_string(), + command: vec!["ls".to_string()], + workdir: "/tmp\nmalicious".to_string(), + ..Default::default() + }; + let err = validate_exec_request_fields(&req).unwrap_err(); + assert!(err.message().contains("newline")); + } + + #[test] + fn validate_exec_still_rejects_newlines_in_env_values() { + let req = ExecSandboxRequest { + sandbox_id: "test".to_string(), + command: vec!["ls".to_string()], + environment: std::iter::once(("VAR".to_string(), "val\nmalicious".to_string())) + .collect(), + ..Default::default() + }; + let err = validate_exec_request_fields(&req).unwrap_err(); + assert!(err.message().contains("newline")); + } } From 94cdd697c55aedb571f177ec13cfa54a8e213919 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jul 2026 06:02:44 +0000 Subject: [PATCH 12/13] chore(deps): bump actions/stale from 10.3.0 to 10.4.0 (#2234) --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 9cf151ea63..5b745057da 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -15,7 +15,7 @@ jobs: issues: write pull-requests: write steps: - - uses: actions/stale@eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899 # v10.3.0 + - uses: actions/stale@1e223db275d687790206a7acac4d1a11bd6fe629 # v10.4.0 with: stale-issue-label: state:stale stale-pr-label: state:stale From d7ffd0a19db1e2475bb5efcef4c2324e328cf722 Mon Sep 17 00:00:00 2001 From: Grace Smith Date: Sun, 12 Jul 2026 21:20:04 +0100 Subject: [PATCH 13/13] fix(certgen): stage temp dir inside output dir to fix cross-device rename generate-certs stages temp files beside --output-dir using dir.with_file_name(). When --output-dir is a container or WSL volume mount, the staging dir lands on a different filesystem and std::fs::rename fails with EXDEV. Move staging inside the output dir so the rename always stays on the same filesystem, preserving atomic replacement. Fixes #2173 Signed-off-by: Grace Smith --- crates/openshell-server/src/certgen.rs | 46 ++++++++++++-------------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/crates/openshell-server/src/certgen.rs b/crates/openshell-server/src/certgen.rs index 683ed180ff..1f3ffca02d 100644 --- a/crates/openshell-server/src/certgen.rs +++ b/crates/openshell-server/src/certgen.rs @@ -624,9 +624,11 @@ fn read_pem(path: &Path) -> Result { } fn write_local_bundle(dir: &Path, bundle: &PkiBundle, paths: &LocalPaths) -> Result<()> { - // Stage to a sibling tmp dir so individual renames into the final layout - // are atomic on the same filesystem. - let temp = sibling_temp_dir(dir); + // Ensure output dir exists before staging inside it. + create_dir_restricted(dir)?; + + // Stage inside the output dir so individual renames are atomic on the same filesystem. + let temp = staging_temp_dir(dir); if temp.exists() { std::fs::remove_dir_all(&temp) .into_diagnostic() @@ -659,8 +661,7 @@ fn write_local_bundle(dir: &Path, bundle: &PkiBundle, paths: &LocalPaths) -> Res )?; write_pem(&temp_jwt.join("kid"), &bundle.jwt_key_id, false)?; - // Final destination (might not exist yet on first run). - create_dir_restricted(dir)?; + // Final destination subdirs (might not exist yet on first run). create_dir_restricted(&paths.server_dir)?; create_dir_restricted(&paths.client_dir)?; create_dir_restricted(&paths.jwt_dir)?; @@ -687,7 +688,10 @@ fn write_local_bundle(dir: &Path, bundle: &PkiBundle, paths: &LocalPaths) -> Res } fn write_local_tls_bundle(bundle: &PkiBundle, paths: &LocalPaths) -> Result<()> { - let temp = sibling_temp_dir(&paths.server_dir); + create_dir_restricted(&paths.server_dir)?; + create_dir_restricted(&paths.client_dir)?; + + let temp = staging_temp_dir(&paths.server_dir); if temp.exists() { std::fs::remove_dir_all(&temp) .into_diagnostic() @@ -707,8 +711,6 @@ fn write_local_tls_bundle(bundle: &PkiBundle, paths: &LocalPaths) -> Result<()> write_pem(&temp_client.join("tls.crt"), &bundle.client_cert_pem, false)?; write_pem(&temp_client.join("tls.key"), &bundle.client_key_pem, true)?; - create_dir_restricted(&paths.server_dir)?; - create_dir_restricted(&paths.client_dir)?; let renames: [(PathBuf, &Path); 6] = [ (temp.join("ca.crt"), paths.ca_crt.as_path()), (temp.join("ca.key"), paths.ca_key.as_path()), @@ -728,7 +730,9 @@ fn write_local_tls_bundle(bundle: &PkiBundle, paths: &LocalPaths) -> Result<()> } fn write_local_jwt_bundle(bundle: &PkiBundle, paths: &LocalPaths) -> Result<()> { - let temp = sibling_temp_dir(&paths.jwt_dir); + create_dir_restricted(&paths.jwt_dir)?; + + let temp = staging_temp_dir(&paths.jwt_dir); if temp.exists() { std::fs::remove_dir_all(&temp) .into_diagnostic() @@ -740,7 +744,6 @@ fn write_local_jwt_bundle(bundle: &PkiBundle, paths: &LocalPaths) -> Result<()> write_pem(&temp.join("public.pem"), &bundle.jwt_public_key_pem, false)?; write_pem(&temp.join("kid"), &bundle.jwt_key_id, false)?; - create_dir_restricted(&paths.jwt_dir)?; let renames: [(PathBuf, &Path); 3] = [ (temp.join("signing.pem"), paths.jwt_signing.as_path()), (temp.join("public.pem"), paths.jwt_public.as_path()), @@ -766,14 +769,9 @@ fn write_pem(path: &Path, contents: &str, owner_only: bool) -> Result<()> { Ok(()) } -fn sibling_temp_dir(dir: &Path) -> PathBuf { - // Use a sibling so std::fs::rename succeeds (same filesystem). - let mut name = dir - .file_name() - .map(std::ffi::OsStr::to_os_string) - .unwrap_or_default(); - name.push(".certgen.tmp"); - dir.with_file_name(name) +fn staging_temp_dir(dir: &Path) -> PathBuf { + // Place temp inside dir so rename stays on the same filesystem as the destination. + dir.join(".certgen.tmp") } // ────────────────────────────── Shared utility ───────────────────────────── @@ -790,7 +788,7 @@ fn print_bundle(bundle: &PkiBundle) { mod tests { use super::{ CertSan, K8sAction, LocalAction, LocalPaths, decide_k8s, decide_local, jwt_signing_secret, - missing_required_server_sans, read_local_bundle, sibling_temp_dir, tls_secret, + missing_required_server_sans, read_local_bundle, staging_temp_dir, tls_secret, write_local_bundle, write_local_jwt_bundle, write_local_tls_bundle, }; use openshell_bootstrap::pki::generate_pki; @@ -902,10 +900,10 @@ mod tests { } #[test] - fn sibling_temp_dir_is_adjacent_to_target() { + fn staging_temp_dir_is_inside_target() { assert_eq!( - sibling_temp_dir(Path::new("/var/lib/openshell/tls")), - Path::new("/var/lib/openshell/tls.certgen.tmp") + staging_temp_dir(Path::new("/var/lib/openshell/tls")), + Path::new("/var/lib/openshell/tls/.certgen.tmp") ); } @@ -922,7 +920,7 @@ mod tests { assert!(f.is_file(), "missing {}", f.display()); } assert!( - !sibling_temp_dir(&dir).exists(), + !staging_temp_dir(&dir).exists(), "temp dir should be cleaned up" ); @@ -1032,7 +1030,7 @@ mod tests { fn write_local_bundle_recovers_from_stale_temp_dir() { let parent = tempfile::tempdir().expect("tempdir"); let dir = parent.path().join("tls"); - let stale = sibling_temp_dir(&dir); + let stale = staging_temp_dir(&dir); std::fs::create_dir_all(&stale).unwrap(); std::fs::write(stale.join("garbage"), "stale").unwrap();