From 5f26d8fa3f9cc701a7cb708a5c0e297374a7ed93 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Tue, 14 Jul 2026 12:32:49 -0700 Subject: [PATCH 01/30] refactor(network): introduce shared egress pipeline Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- architecture/sandbox.md | 11 + .../openshell-supervisor-network/src/proxy.rs | 1324 ++++++----------- .../src/proxy/destination.rs | 197 +++ .../src/proxy/egress.rs | 144 ++ .../src/proxy/relay.rs | 149 ++ 5 files changed, 987 insertions(+), 838 deletions(-) create mode 100644 crates/openshell-supervisor-network/src/proxy/destination.rs create mode 100644 crates/openshell-supervisor-network/src/proxy/egress.rs create mode 100644 crates/openshell-supervisor-network/src/proxy/relay.rs diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 4f7dfe4272..95be47569a 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -57,6 +57,17 @@ unsafe internal destinations, and evaluates the active policy. On Linux, it maps an accepted proxy connection back to the workload socket by matching the complete local-to-remote TCP tuple before resolving every process that owns the socket inode. + +CONNECT and absolute-form forward HTTP are explicit-proxy adapters over the same +egress pipeline. Each adapter normalizes its request into an egress intent, and +the shared authorization result carries the process evidence and endpoint state +used by destination validation and relay selection. Destination validation +returns an unopened connector so adapters retain their existing response and +upstream-dial timing. CONNECT uses shared TLS-terminated HTTP, plaintext HTTP, +and raw byte relay primitives. Forward HTTP retains its guarded single-request +relay while sharing authorization, request context, and destination boundaries. +Adapter-specific response and OCSF event shapes remain at the protocol boundary. + For inspected HTTP traffic, the proxy can enforce REST method/path rules, WebSocket upgrade and text-message rules, GraphQL operation rules, and MCP method, tool, and supported params rules or generic JSON-RPC method rules diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 6e9c48220b..b02fda9a98 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -3,6 +3,10 @@ //! HTTP CONNECT proxy with OPA policy evaluation and process-identity binding. +mod destination; +mod egress; +mod relay; + use crate::identity::BinaryIdentityCache; use crate::l7::tls::ProxyTlsState; use crate::opa::{NetworkAction, OpaEngine, PolicyGenerationGuard}; @@ -31,6 +35,14 @@ use tokio::sync::mpsc; use tokio::task::JoinHandle; use tracing::{debug, warn}; +use self::destination::{ + DestinationDenial, DestinationDenialKind, DestinationRequest, validate_destination, +}; +use self::egress::{ + EgressDecision, EgressIntent, EndpointDecision, IdentityUnavailableReason, L7ConfigSnapshot, + L7RouteSnapshot, ProcessIdentityEvidence, +}; + const MAX_HEADER_BYTES: usize = 8192; const TUNNEL_PROTOCOL_PEEK_BYTES: usize = crate::l7::rest::HTTP2_PRIOR_KNOWLEDGE_PREFACE.len(); #[cfg(not(test))] @@ -82,21 +94,6 @@ const CHUNK_IDLE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(1 #[cfg(test)] const CHUNK_IDLE_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(100); -/// Result of a proxy CONNECT policy decision. -struct ConnectDecision { - action: NetworkAction, - /// Policy generation used for the L4 network decision. - generation: u64, - /// Resolved binary path. - binary: Option, - /// PID owning the socket. - binary_pid: Option, - /// Ancestor binary paths from process tree walk. - ancestors: Vec, - /// Cmdline-derived absolute paths (for script detection). - cmdline_paths: Vec, -} - /// Outcome of an inference interception attempt. /// /// Returned by [`handle_inference_interception`] so the call site can emit @@ -758,7 +755,7 @@ fn emit_denial( host: &str, port: u16, binary: &str, - decision: &ConnectDecision, + decision: &EgressDecision, reason: &str, stage: &str, ) { @@ -787,7 +784,7 @@ fn emit_denial_simple( host: &str, port: u16, binary: &str, - decision: &ConnectDecision, + decision: &EgressDecision, reason: &str, stage: &str, ) { @@ -809,6 +806,151 @@ fn emit_denial_simple( } } +#[allow(clippy::too_many_arguments)] +async fn deny_connect_destination( + client: &mut TcpStream, + denial: &DestinationDenial, + peer_addr: SocketAddr, + host: &str, + port: u16, + binary: &str, + pid: &str, + ancestors: &str, + cmdline: &str, + decision: &EgressDecision, + denial_tx: &Option>, + activity_tx: &Option, +) -> Result<()> { + let detail = match denial.kind { + DestinationDenialKind::TrustedGateway => "trusted-gateway check failed", + DestinationDenialKind::InvalidAllowedIps => "invalid allowed_ips in policy", + DestinationDenialKind::AllowedIps => "allowed_ips check failed", + DestinationDenialKind::DeclaredEndpoint => "declared endpoint check failed", + DestinationDenialKind::InternalAddress => "internal address", + }; + let message = if denial.kind == DestinationDenialKind::InternalAddress { + format!("CONNECT blocked: internal address {host}:{port}") + } else { + format!("CONNECT blocked: {detail} for {host}:{port}") + }; + + let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Open) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .dst_endpoint(Endpoint::from_domain(host, port)) + .src_endpoint_addr(peer_addr.ip(), peer_addr.port()) + .actor_process(Process::from_bypass(binary, pid, ancestors).with_cmd_line(cmdline)) + .firewall_rule("-", "ssrf") + .message(message) + .status_detail(&denial.reason) + .build(); + ocsf_emit!(event); + + emit_denial( + denial_tx, + host, + port, + binary, + decision, + &denial.reason, + "ssrf", + ); + // Preserve the current activity contract. The declared-endpoint branch + // historically emits the denial without a separate SSRF activity count. + if denial.kind != DestinationDenialKind::DeclaredEndpoint { + emit_activity(activity_tx, true, "ssrf"); + } + respond( + client, + &build_json_error_response( + 403, + "Forbidden", + "ssrf_denied", + &format!("CONNECT {host}:{port} blocked: {detail}"), + ), + ) + .await +} + +#[allow(clippy::too_many_arguments)] +async fn deny_forward_destination( + client: &mut TcpStream, + denial: &DestinationDenial, + peer_addr: SocketAddr, + method: &str, + host: &str, + port: u16, + path: &str, + binary: &str, + pid: &str, + ancestors: &str, + cmdline: &str, + policy: &str, + decision: &EgressDecision, + denial_tx: Option<&mpsc::UnboundedSender>, + activity_tx: Option<&ActivitySender>, +) -> Result<()> { + let detail = match denial.kind { + DestinationDenialKind::TrustedGateway => "trusted-gateway check failed", + DestinationDenialKind::InvalidAllowedIps => "invalid allowed_ips in policy", + DestinationDenialKind::AllowedIps => "allowed_ips check failed", + DestinationDenialKind::DeclaredEndpoint => "declared endpoint check failed", + DestinationDenialKind::InternalAddress => "internal address", + }; + let log_detail = if denial.kind == DestinationDenialKind::InternalAddress { + "internal IP without allowed_ips" + } else { + detail + }; + + let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Other) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .http_request(HttpRequest::new( + method, + OcsfUrl::new("http", host, path, port), + )) + .dst_endpoint(Endpoint::from_domain(host, port)) + .src_endpoint(Endpoint::from_ip(peer_addr.ip(), peer_addr.port())) + .actor_process(Process::from_bypass(binary, pid, ancestors).with_cmd_line(cmdline)) + .firewall_rule(policy, "ssrf") + .message(format!("FORWARD blocked: {log_detail} for {host}:{port}")) + .status_detail(&denial.reason) + .build(); + ocsf_emit!(event); + + emit_denial_simple( + denial_tx, + host, + port, + binary, + decision, + &denial.reason, + "ssrf", + ); + // Preserve the current activity contract. The declared-endpoint branch + // historically emits the denial without a separate SSRF activity count. + if denial.kind != DestinationDenialKind::DeclaredEndpoint { + emit_activity_simple(activity_tx, true, "ssrf"); + } + respond( + client, + &build_json_error_response( + 403, + "Forbidden", + "ssrf_denied", + &format!("{method} {host}:{port} blocked: {detail}"), + ), + ) + .await +} + // Many distinct, non-related context parameters are required for a CONNECT // dispatch; bundling them into a struct would just shift the noise into call // sites. @@ -941,20 +1083,19 @@ async fn handle_tcp_connection( let opa_clone = opa_engine.clone(); let cache_clone = identity_cache.clone(); let pid_clone = entrypoint_pid.clone(); - let host_clone = host_lc.clone(); - let decision = tokio::task::spawn_blocking(move || { - evaluate_opa_tcp( - connection, - &opa_clone, - &cache_clone, - &pid_clone, - &host_clone, - port, - ) + let intent = EgressIntent::connect(host_lc.clone(), port); + let mut decision = tokio::task::spawn_blocking(move || { + evaluate_opa_tcp(connection, &opa_clone, &cache_clone, &pid_clone, intent) }) .await .map_err(|e| miette::miette!("identity resolution task panicked: {e}"))?; + debug!( + transport = ?decision.intent.transport, + identity = ?decision.identity, + "Authorized explicit proxy egress intent" + ); + // Extract action string and matched policy for logging let (matched_policy, deny_reason) = match &decision.action { NetworkAction::Allow { matched_policy } => (matched_policy.clone(), String::new()), @@ -1042,288 +1183,44 @@ async fn handle_tcp_connection( // allowed_ips validation below — so an internal-address CONNECT still gets // the SSRF 403 and telemetry in degraded state — but before the upstream // connect and before `200 Connection Established`. - let effective_tls_skip = - query_tls_mode(&opa_engine, &decision, &host_lc, port) == crate::l7::TlsMode::Skip; + hydrate_tls_mode(&opa_engine, &mut decision); + let effective_tls_skip = decision.endpoint.tls_mode == crate::l7::TlsMode::Skip; let sandbox_entrypoint_pid = entrypoint_pid.load(Ordering::Acquire); - // Query allowed_ips from the matched endpoint config (if any). - // When present, the SSRF check validates resolved IPs against this - // allowlist instead of blanket-blocking all private IPs. - // When the policy host is already a literal IP address, treat it as - // implicitly allowed — the user explicitly declared the destination. - // Exact declared hostnames also skip the private-IP blanket block below, - // while keeping loopback/link-local/unspecified addresses denied. - let mut raw_allowed_ips = query_allowed_ips(&opa_engine, &decision, &host_lc, port); - if raw_allowed_ips.is_empty() { - raw_allowed_ips = implicit_allowed_ips_for_ip_host(&host); - } - let exact_declared_endpoint_host = - query_exact_declared_endpoint_host(&opa_engine, &decision, &host_lc, port); + hydrate_destination_constraints(&opa_engine, &mut decision); // Defense-in-depth: resolve DNS and reject connections to internal IPs. let dns_connect_start = std::time::Instant::now(); - // The "non-empty" branch is the explicit-allowlist path; reading it first - // matches the policy decision narrative. - #[allow(clippy::if_not_else)] - let validated_addrs = if is_host_gateway_alias(&host_lc) - && let Some(gw) = *trusted_host_gateway + let connector = match validate_destination(DestinationRequest { + host: &host, + normalized_host: &host_lc, + port, + sandbox_entrypoint_pid, + trusted_host_gateway: *trusted_host_gateway, + raw_allowed_ips: decision.endpoint.raw_allowed_ips.clone(), + exact_declared_endpoint_host: decision.endpoint.exact_declared_host, + }) + .await { - // Trusted host-gateway path. The compute driver injected this hostname - // into /etc/hosts pointing at a known IP (read at proxy startup before - // user code runs). Bypass the normal SSRF tiers so link-local gateway - // addresses (used by rootless Podman with pasta) are not hard-blocked. - // Cloud metadata IPs and control-plane ports are still rejected. - match resolve_and_check_trusted_gateway(&host, port, gw, sandbox_entrypoint_pid).await { - Ok(addrs) => addrs, - Err(reason) => { - { - let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Open) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .src_endpoint_addr(workload_addr.ip(), workload_addr.port()) - .actor_process( - Process::from_bypass(&binary_str, &pid_str, &ancestors_str) - .with_cmd_line(&cmdline_str), - ) - .firewall_rule("-", "ssrf") - .message(format!( - "CONNECT blocked: trusted-gateway check failed for {host_lc}:{port}" - )) - .status_detail(&reason) - .build(); - ocsf_emit!(event); - } - emit_denial( - &denial_tx, - &host_lc, - port, - &binary_str, - &decision, - &reason, - "ssrf", - ); - emit_activity(&activity_tx, true, "ssrf"); - respond( - &mut client, - &build_json_error_response( - 403, - "Forbidden", - "ssrf_denied", - &format!("CONNECT {host_lc}:{port} blocked: trusted-gateway check failed"), - ), - ) - .await?; - return Ok(()); - } - } - } else if !raw_allowed_ips.is_empty() { - // allowed_ips mode: validate resolved IPs against CIDR allowlist. - // Loopback and link-local are still always blocked. - match parse_allowed_ips(&raw_allowed_ips) { - Ok(nets) => { - match resolve_and_check_allowed_ips(&host, port, &nets, sandbox_entrypoint_pid) - .await - { - Ok(addrs) => addrs, - Err(reason) => { - { - let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Open) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .src_endpoint_addr(workload_addr.ip(), workload_addr.port()) - .actor_process( - Process::from_bypass(&binary_str, &pid_str, &ancestors_str) - .with_cmd_line(&cmdline_str), - ) - .firewall_rule("-", "ssrf") - .message(format!( - "CONNECT blocked: allowed_ips check failed for {host_lc}:{port}" - )) - .status_detail(&reason) - .build(); - ocsf_emit!(event); - } - emit_denial( - &denial_tx, - &host_lc, - port, - &binary_str, - &decision, - &reason, - "ssrf", - ); - emit_activity(&activity_tx, true, "ssrf"); - respond( - &mut client, - &build_json_error_response( - 403, - "Forbidden", - "ssrf_denied", - &format!( - "CONNECT {host_lc}:{port} blocked: allowed_ips check failed" - ), - ), - ) - .await?; - return Ok(()); - } - } - } - Err(reason) => { - { - let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Open) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .src_endpoint_addr(workload_addr.ip(), workload_addr.port()) - .actor_process( - Process::from_bypass(&binary_str, &pid_str, &ancestors_str) - .with_cmd_line(&cmdline_str), - ) - .firewall_rule("-", "ssrf") - .message(format!( - "CONNECT blocked: invalid allowed_ips in policy for {host_lc}:{port}" - )) - .status_detail(&reason) - .build(); - ocsf_emit!(event); - } - emit_denial( - &denial_tx, - &host_lc, - port, - &binary_str, - &decision, - &reason, - "ssrf", - ); - emit_activity(&activity_tx, true, "ssrf"); - respond( - &mut client, - &build_json_error_response( - 403, - "Forbidden", - "ssrf_denied", - &format!("CONNECT {host_lc}:{port} blocked: invalid allowed_ips in policy"), - ), - ) - .await?; - return Ok(()); - } - } - } else if exact_declared_endpoint_host { - // Exact declared hostname mode: the operator explicitly allowed this - // host:port, so private IP resolution is permitted without duplicating - // the resolved IP in allowed_ips. Always-blocked addresses and - // control-plane ports remain denied. - match resolve_and_check_declared_endpoint(&host, port, sandbox_entrypoint_pid).await { - Ok(addrs) => addrs, - Err(reason) => { - { - let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Open) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .src_endpoint_addr(workload_addr.ip(), workload_addr.port()) - .actor_process( - Process::from_bypass(&binary_str, &pid_str, &ancestors_str) - .with_cmd_line(&cmdline_str), - ) - .firewall_rule("-", "ssrf") - .message(format!( - "CONNECT blocked: declared endpoint check failed for {host_lc}:{port}" - )) - .status_detail(&reason) - .build(); - ocsf_emit!(event); - } - emit_denial( - &denial_tx, - &host_lc, - port, - &binary_str, - &decision, - &reason, - "ssrf", - ); - respond( - &mut client, - &build_json_error_response( - 403, - "Forbidden", - "ssrf_denied", - &format!( - "CONNECT {host_lc}:{port} blocked: declared endpoint check failed" - ), - ), - ) - .await?; - return Ok(()); - } - } - } else { - // Default: reject all internal IPs (loopback, RFC 1918, link-local). - match resolve_and_reject_internal(&host, port, sandbox_entrypoint_pid).await { - Ok(addrs) => addrs, - Err(reason) => { - { - let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Open) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .src_endpoint_addr(workload_addr.ip(), workload_addr.port()) - .actor_process( - Process::from_bypass(&binary_str, &pid_str, &ancestors_str) - .with_cmd_line(&cmdline_str), - ) - .firewall_rule("-", "ssrf") - .message(format!( - "CONNECT blocked: internal address {host_lc}:{port}" - )) - .status_detail(&reason) - .build(); - ocsf_emit!(event); - } - emit_denial( - &denial_tx, - &host_lc, - port, - &binary_str, - &decision, - &reason, - "ssrf", - ); - emit_activity(&activity_tx, true, "ssrf"); - respond( - &mut client, - &build_json_error_response( - 403, - "Forbidden", - "ssrf_denied", - &format!("CONNECT {host_lc}:{port} blocked: internal address"), - ), - ) - .await?; - return Ok(()); - } + Ok(connector) => connector, + Err(denial) => { + deny_connect_destination( + &mut client, + &denial, + workload_addr, + &host_lc, + port, + &binary_str, + &pid_str, + &ancestors_str, + &cmdline_str, + &decision, + &denial_tx, + &activity_tx, + ) + .await?; + return Ok(()); } }; @@ -1369,9 +1266,14 @@ async fn handle_tcp_connection( return Ok(()); } - let mut upstream = dial_upstream(&upstream_proxy, &host_lc, port, &validated_addrs) - .await - .into_diagnostic()?; + let mut upstream = dial_upstream( + &upstream_proxy, + &host_lc, + port, + connector.addrs(), + ) + .await + .into_diagnostic()?; debug!( "handle_tcp_connection dns_resolve_and_tcp_connect: {}ms host={host_lc}", @@ -1382,8 +1284,9 @@ async fn handle_tcp_connection( // Check if endpoint has L7 config for protocol-aware inspection, and // retain the generation for HTTP passthrough keep-alive tunnels. - let l7_route = query_l7_route_snapshot(&opa_engine, &decision, &host_lc, port); - let should_inspect_l7 = l7_inspection_active(l7_route.as_ref()); + hydrate_l7_route(&opa_engine, &mut decision); + let l7_route = decision.endpoint.l7_route.as_ref(); + let should_inspect_l7 = l7_inspection_active(l7_route); // Log the allowed CONNECT — use CONNECT_L7 when L7 inspection follows, // so log consumers can distinguish L4-only decisions from tunnel lifecycle events. @@ -1410,39 +1313,19 @@ async fn handle_tcp_connection( .build(); ocsf_emit!(event); } - emit_connect_activity_if_l4_only(&activity_tx, l7_route.as_ref()); + emit_connect_activity_if_l4_only(&activity_tx, l7_route); // `effective_tls_skip` was resolved before the `200` above (the fail-closed // gate needs it) and drives the raw-tunnel branch below. - // Build L7 eval context (shared by TLS-terminated and plaintext paths). - let ctx = crate::l7::relay::L7EvalContext { - host: host_lc.clone(), - port, - policy_name: matched_policy.clone().unwrap_or_default(), - binary_path: decision - .binary - .as_ref() - .map(|p| p.to_string_lossy().into_owned()) - .unwrap_or_default(), - ancestors: decision - .ancestors - .iter() - .map(|p| p.to_string_lossy().into_owned()) - .collect(), - cmdline_paths: decision - .cmdline_paths - .iter() - .map(|p| p.to_string_lossy().into_owned()) - .collect(), - secret_resolver: secret_resolver.clone(), - activity_tx: activity_tx.clone(), - dynamic_credentials: dynamic_credentials.clone(), - token_grant_resolver: dynamic_credentials - .as_ref() - .map(|_| crate::l7::token_grant_injection::default_resolver()), + // Build request-processing context shared by CONNECT and forward HTTP. + let ctx = relay::http_context( + &decision, + secret_resolver.clone(), + activity_tx.clone(), + dynamic_credentials.clone(), agent_proposals, - }; + ); if effective_tls_skip { // Policy validation rejects fail-closed middleware overlapping @@ -1477,9 +1360,7 @@ async fn handle_tcp_connection( port = port, "tls: skip — bypassing TLS auto-detection, raw tunnel" ); - let _ = tokio::io::copy_bidirectional(&mut client, &mut upstream) - .await - .into_diagnostic()?; + relay::relay_tcp(&mut client, &mut upstream).await?; return Ok(()); } @@ -1500,60 +1381,15 @@ async fn handle_tcp_connection( crate::l7::tls::tls_connect_upstream(upstream, &host_lc, tls.upstream_config()) .await?; - if let Some(route) = l7_route.as_ref().filter(|route| !route.configs.is_empty()) { - // L7 inspection on terminated TLS traffic. - let tunnel_engine = match opa_engine.clone_engine_for_tunnel(route.generation) { - Ok(engine) => engine, - Err(e) => { - emit_l7_tunnel_close_after_policy_change(&host_lc, port, e); - return Ok(()); - } - }; - if route.configs.len() == 1 { - crate::l7::relay::relay_with_inspection( - &route.configs[0].config, - tunnel_engine, - &mut tls_client, - &mut tls_upstream, - &ctx, - ) - .await - } else { - let configs: Vec = route - .configs - .iter() - .map(|snapshot| snapshot.config.clone()) - .collect(); - crate::l7::relay::relay_with_route_selection( - &configs, - tunnel_engine, - &mut tls_client, - &mut tls_upstream, - &ctx, - ) - .await - } - } else { - // No L7 config — relay with credential injection only. - let generation = l7_route - .as_ref() - .map_or(decision.generation, |route| route.generation); - let generation_guard = match opa_engine.generation_guard(generation) { - Ok(guard) => guard, - Err(e) => { - emit_l7_tunnel_close_after_policy_change(&host_lc, port, e); - return Ok(()); - } - }; - crate::l7::relay::relay_passthrough_with_credentials( - &mut tls_client, - &mut tls_upstream, - &ctx, - &generation_guard, - Some(&opa_engine), - ) - .await - } + relay::relay_http_stream( + l7_route, + &opa_engine, + &decision, + &mut tls_client, + &mut tls_upstream, + &ctx, + ) + .await }; if let Err(e) = tls_result.await { if is_benign_relay_error(&e) { @@ -1620,85 +1456,37 @@ async fn handle_tcp_connection( } } else if tunnel_protocol == TunnelProtocol::Http1 { // Plaintext HTTP detected. - if let Some(route) = l7_route.as_ref().filter(|route| !route.configs.is_empty()) { - let tunnel_engine = match opa_engine.clone_engine_for_tunnel(route.generation) { - Ok(engine) => engine, - Err(e) => { - emit_l7_tunnel_close_after_policy_change(&host_lc, port, e); - return Ok(()); - } - }; - let relay_result = if route.configs.len() == 1 { - crate::l7::relay::relay_with_inspection( - &route.configs[0].config, - tunnel_engine, - &mut client, - &mut upstream, - &ctx, - ) - .await - } else { - let configs: Vec = route - .configs - .iter() - .map(|snapshot| snapshot.config.clone()) - .collect(); - crate::l7::relay::relay_with_route_selection( - &configs, - tunnel_engine, - &mut client, - &mut upstream, - &ctx, - ) - .await - }; - if let Err(e) = relay_result { - if is_benign_relay_error(&e) { + let is_l7_relay = l7_route.is_some_and(|route| !route.configs.is_empty()); + if let Err(e) = relay::relay_http_stream( + l7_route, + &opa_engine, + &decision, + &mut client, + &mut upstream, + &ctx, + ) + .await + { + if is_benign_relay_error(&e) { + if is_l7_relay { debug!(host = %host_lc, port = port, error = %e, "L7 connection closed"); } else { - let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::Low) - .status(StatusId::Failure) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .message(format!("L7 relay error: {e}")) - .build(); - ocsf_emit!(event); - } - } - } else { - // Plaintext HTTP, no L7 config — relay with credential injection. - let generation = l7_route - .as_ref() - .map_or(decision.generation, |route| route.generation); - let generation_guard = match opa_engine.generation_guard(generation) { - Ok(guard) => guard, - Err(e) => { - emit_l7_tunnel_close_after_policy_change(&host_lc, port, e); - return Ok(()); - } - }; - if let Err(e) = crate::l7::relay::relay_passthrough_with_credentials( - &mut client, - &mut upstream, - &ctx, - &generation_guard, - Some(&opa_engine), - ) - .await - { - if is_benign_relay_error(&e) { debug!(host = %host_lc, port = port, error = %e, "HTTP relay closed"); - } else { - let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::Low) - .status(StatusId::Failure) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .message(format!("HTTP relay error: {e}")) - .build(); - ocsf_emit!(event); } + } else { + let message = if is_l7_relay { + format!("L7 relay error: {e}") + } else { + format!("HTTP relay error: {e}") + }; + let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::Low) + .status(StatusId::Failure) + .dst_endpoint(Endpoint::from_domain(&host_lc, port)) + .message(message) + .build(); + ocsf_emit!(event); } } } else { @@ -1761,9 +1549,7 @@ async fn handle_tcp_connection( port = port, "Non-TLS non-HTTP traffic detected, raw tunnel" ); - let _ = tokio::io::copy_bidirectional(&mut client, &mut upstream) - .await - .into_diagnostic()?; + relay::relay_tcp(&mut client, &mut upstream).await?; } Ok(()) @@ -1806,7 +1592,7 @@ impl ResolvedIdentity { /// Error from [`resolve_process_identity`]. Carries the deny reason and /// whatever partial identity data was resolved before the failure so the -/// caller can include it in the [`ConnectDecision`] and OCSF event. +/// caller can include it in the [`EgressDecision`] and OCSF event. #[cfg(target_os = "linux")] struct IdentityError { reason: String, @@ -1990,21 +1776,24 @@ fn evaluate_opa_tcp( engine: &OpaEngine, identity_cache: &BinaryIdentityCache, entrypoint_pid: &AtomicU32, - host: &str, - port: u16, -) -> ConnectDecision { + intent: EgressIntent, +) -> EgressDecision { use crate::opa::NetworkInput; use std::sync::atomic::Ordering; let deny = |reason: String, + identity: ProcessIdentityEvidence, binary: Option, binary_pid: Option, ancestors: Vec, cmdline_paths: Vec| - -> ConnectDecision { - ConnectDecision { + -> EgressDecision { + EgressDecision { + intent: intent.clone(), action: NetworkAction::Deny { reason }, generation: engine.current_generation(), + identity, + endpoint: EndpointDecision::default(), binary, binary_pid, ancestors, @@ -2013,9 +1802,12 @@ fn evaluate_opa_tcp( }; if !crate::opa::network_binary_identity_required() { - let result = evaluate_endpoint_only_opa(engine, host, port); + let result = evaluate_endpoint_only_opa(engine, intent); debug!( - "evaluate_opa_tcp endpoint-only: host={host} port={port} action={:?}", + "evaluate_opa_tcp endpoint-only: host={} port={} transport={:?} action={:?}", + result.intent.destination.host, + result.intent.destination.port, + result.intent.transport, result.action ); return result; @@ -2025,6 +1817,7 @@ fn evaluate_opa_tcp( let Some(proc_net_anchor_pid) = proc_net_anchor_pid(entrypoint_pid) else { return deny( "entrypoint process not yet spawned".into(), + ProcessIdentityEvidence::Unavailable(IdentityUnavailableReason::LookupFailed), None, None, vec![], @@ -2038,6 +1831,7 @@ fn evaluate_opa_tcp( Err(err) => { return deny( err.reason, + ProcessIdentityEvidence::Unavailable(IdentityUnavailableReason::LookupFailed), err.binary, err.binary_pid, err.ancestors, @@ -2055,8 +1849,8 @@ fn evaluate_opa_tcp( } = identity; let input = NetworkInput { - host: host.to_string(), - port, + host: intent.destination.host.clone(), + port: intent.destination.port, binary_path: bin_path.clone(), binary_sha256: bin_hash, ancestors: ancestors.clone(), @@ -2064,9 +1858,12 @@ fn evaluate_opa_tcp( }; let result = match engine.evaluate_network_action_with_generation(&input) { - Ok((action, generation)) => ConnectDecision { + Ok((action, generation)) => EgressDecision { + intent: intent.clone(), action, generation, + identity: ProcessIdentityEvidence::Available, + endpoint: EndpointDecision::default(), binary: Some(bin_path), binary_pid: Some(binary_pid), ancestors, @@ -2074,6 +1871,7 @@ fn evaluate_opa_tcp( }, Err(e) => deny( format!("policy evaluation error: {e}"), + ProcessIdentityEvidence::Available, Some(bin_path), Some(binary_pid), ancestors, @@ -2081,8 +1879,11 @@ fn evaluate_opa_tcp( ), }; debug!( - "evaluate_opa_tcp TOTAL: {}ms host={host} port={port}", - total_start.elapsed().as_millis() + "evaluate_opa_tcp TOTAL: {}ms host={} port={} transport={:?}", + total_start.elapsed().as_millis(), + intent.destination.host, + intent.destination.port, + intent.transport, ); result } @@ -2101,10 +1902,10 @@ fn sidecar_topology_enabled() -> bool { .is_ok_and(|value| value == SIDECAR_SUPERVISOR_TOPOLOGY) } -fn evaluate_endpoint_only_opa(engine: &OpaEngine, host: &str, port: u16) -> ConnectDecision { +fn evaluate_endpoint_only_opa(engine: &OpaEngine, intent: EgressIntent) -> EgressDecision { let input = crate::opa::NetworkInput { - host: host.to_string(), - port, + host: intent.destination.host.clone(), + port: intent.destination.port, binary_path: PathBuf::new(), binary_sha256: String::new(), ancestors: vec![], @@ -2112,19 +1913,29 @@ fn evaluate_endpoint_only_opa(engine: &OpaEngine, host: &str, port: u16) -> Conn }; match engine.evaluate_network_action_with_generation(&input) { - Ok((action, generation)) => ConnectDecision { + Ok((action, generation)) => EgressDecision { + intent, action, generation, + identity: ProcessIdentityEvidence::Unavailable( + IdentityUnavailableReason::EndpointOnlyMode, + ), + endpoint: EndpointDecision::default(), binary: None, binary_pid: None, ancestors: vec![], cmdline_paths: vec![], }, - Err(e) => ConnectDecision { + Err(e) => EgressDecision { + intent, action: NetworkAction::Deny { reason: format!("policy evaluation error: {e}"), }, generation: engine.current_generation(), + identity: ProcessIdentityEvidence::Unavailable( + IdentityUnavailableReason::EndpointOnlyMode, + ), + endpoint: EndpointDecision::default(), binary: None, binary_pid: None, ancestors: vec![], @@ -2140,18 +1951,22 @@ fn evaluate_opa_tcp( engine: &OpaEngine, _identity_cache: &BinaryIdentityCache, _entrypoint_pid: &AtomicU32, - host: &str, - port: u16, -) -> ConnectDecision { + intent: EgressIntent, +) -> EgressDecision { if !crate::opa::network_binary_identity_required() { - return evaluate_endpoint_only_opa(engine, host, port); + return evaluate_endpoint_only_opa(engine, intent); } - ConnectDecision { + EgressDecision { + intent, action: NetworkAction::Deny { reason: "identity binding unavailable on this platform".into(), }, generation: engine.current_generation(), + identity: ProcessIdentityEvidence::Unavailable( + IdentityUnavailableReason::UnsupportedPlatform, + ), + endpoint: EndpointDecision::default(), binary: None, binary_pid: None, ancestors: vec![], @@ -2593,17 +2408,6 @@ async fn write_all(writer: &mut (impl tokio::io::AsyncWrite + Unpin), data: &[u8 Ok(()) } -#[derive(Debug, Clone)] -struct L7ConfigSnapshot { - config: crate::l7::L7EndpointConfig, -} - -#[derive(Debug, Clone)] -struct L7RouteSnapshot { - configs: Vec, - generation: u64, -} - fn emit_l7_tunnel_close_after_policy_change(host: &str, port: u16, error: miette::Report) { let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Open) @@ -2623,9 +2427,29 @@ fn emit_l7_tunnel_close_after_policy_change(host: &str, port: u16, error: miette /// /// Returns `Some(L7EndpointConfig)` if the matched endpoint has L7 config (protocol field), /// `None` for L4-only endpoints. +fn hydrate_l7_route(engine: &OpaEngine, decision: &mut EgressDecision) { + let host = decision.intent.destination.host.clone(); + let port = decision.intent.destination.port; + decision.endpoint.l7_route = query_l7_route_snapshot(engine, decision, &host, port); +} + +fn hydrate_tls_mode(engine: &OpaEngine, decision: &mut EgressDecision) { + let host = decision.intent.destination.host.clone(); + let port = decision.intent.destination.port; + decision.endpoint.tls_mode = query_tls_mode(engine, decision, &host, port); +} + +fn hydrate_destination_constraints(engine: &OpaEngine, decision: &mut EgressDecision) { + let host = decision.intent.destination.host.clone(); + let port = decision.intent.destination.port; + decision.endpoint.raw_allowed_ips = query_allowed_ips(engine, decision, &host, port); + decision.endpoint.exact_declared_host = + query_exact_declared_endpoint_host(engine, decision, &host, port); +} + fn query_l7_route_snapshot( engine: &OpaEngine, - decision: &ConnectDecision, + decision: &EgressDecision, host: &str, port: u16, ) -> Option { @@ -2695,7 +2519,7 @@ fn select_l7_config_for_path<'a>( /// This extracts `tls: skip` from the endpoint even when no `protocol` is set. fn query_tls_mode( engine: &OpaEngine, - decision: &ConnectDecision, + decision: &EgressDecision, host: &str, port: u16, ) -> crate::l7::TlsMode { @@ -3296,7 +3120,7 @@ fn parse_allowed_ips(raw: &[String]) -> std::result::Result, S /// Query `allowed_ips` from the matched endpoint config for a CONNECT decision. fn query_allowed_ips( engine: &OpaEngine, - decision: &ConnectDecision, + decision: &EgressDecision, host: &str, port: u16, ) -> Vec { @@ -3339,7 +3163,7 @@ fn query_allowed_ips( /// Query whether the matched endpoint was declared as this exact hostname. fn query_exact_declared_endpoint_host( engine: &OpaEngine, - decision: &ConnectDecision, + decision: &EgressDecision, host: &str, port: u16, ) -> bool { @@ -3912,20 +3736,19 @@ async fn handle_forward_proxy( let opa_clone = opa_engine.clone(); let cache_clone = identity_cache.clone(); let pid_clone = entrypoint_pid.clone(); - let host_clone = host_lc.clone(); - let decision = tokio::task::spawn_blocking(move || { - evaluate_opa_tcp( - connection, - &opa_clone, - &cache_clone, - &pid_clone, - &host_clone, - port, - ) + let intent = EgressIntent::forward_http(host_lc.clone(), port); + let mut decision = tokio::task::spawn_blocking(move || { + evaluate_opa_tcp(connection, &opa_clone, &cache_clone, &pid_clone, intent) }) .await .map_err(|e| miette::miette!("identity resolution task panicked: {e}"))?; + debug!( + transport = ?decision.intent.transport, + identity = ?decision.identity, + "Authorized explicit proxy egress intent" + ); + // Build log context let binary_str = decision .binary @@ -4057,40 +3880,24 @@ async fn handle_forward_proxy( let mut forward_websocket_request = crate::l7::rest::request_is_websocket_upgrade(&forward_request_bytes); let mut request_body_credential_rewrite = false; - let l7_ctx = crate::l7::relay::L7EvalContext { - host: host_lc.clone(), - port, - policy_name: matched_policy.clone().unwrap_or_default(), - binary_path: decision - .binary - .as_ref() - .map(|p| p.to_string_lossy().into_owned()) - .unwrap_or_default(), - ancestors: decision - .ancestors - .iter() - .map(|p| p.to_string_lossy().into_owned()) - .collect(), - cmdline_paths: decision - .cmdline_paths - .iter() - .map(|p| p.to_string_lossy().into_owned()) - .collect(), - secret_resolver: secret_resolver.clone(), - activity_tx: activity_tx.cloned(), - dynamic_credentials: dynamic_credentials.clone(), - token_grant_resolver: dynamic_credentials - .as_ref() - .map(|_| crate::l7::token_grant_injection::default_resolver()), + let l7_ctx = relay::http_context( + &decision, + secret_resolver.clone(), + activity_tx.cloned(), + dynamic_credentials.clone(), agent_proposals, - }; + ); let mut l7_activity_pending = false; // 4b. If the endpoint has L7 config, evaluate the request against // L7 policy. The forward proxy handles exactly one request per // connection (Connection: close), so a single evaluation suffices. - if let Some(route) = query_l7_route_snapshot(&opa_engine, &decision, &host_lc, port) - && !route.configs.is_empty() + hydrate_l7_route(&opa_engine, &mut decision); + if let Some(route) = decision + .endpoint + .l7_route + .as_ref() + .filter(|route| !route.configs.is_empty()) { if route.generation != forward_generation_guard.captured_generation() { warn!( @@ -4527,293 +4334,40 @@ async fn handle_forward_proxy( // - Otherwise: reject internal IPs, allow public IPs through. // When the policy host is already a literal IP address, treat it as // implicitly allowed — the user explicitly declared the destination. - let mut raw_allowed_ips = query_allowed_ips(&opa_engine, &decision, &host_lc, port); - if raw_allowed_ips.is_empty() { - raw_allowed_ips = implicit_allowed_ips_for_ip_host(&host); - } - let exact_declared_endpoint_host = - query_exact_declared_endpoint_host(&opa_engine, &decision, &host_lc, port); - - // The trusted-gateway branch is the first path; reading it before the - // allowed_ips and default branches matches the policy decision narrative. - #[allow(clippy::if_not_else)] - let addrs = if is_host_gateway_alias(&host_lc) - && let Some(gw) = *trusted_host_gateway + hydrate_destination_constraints(&opa_engine, &mut decision); + + let connector = match validate_destination(DestinationRequest { + host: &host, + normalized_host: &host_lc, + port, + sandbox_entrypoint_pid, + trusted_host_gateway: *trusted_host_gateway, + raw_allowed_ips: decision.endpoint.raw_allowed_ips.clone(), + exact_declared_endpoint_host: decision.endpoint.exact_declared_host, + }) + .await { - // Trusted host-gateway path. Mirrors the CONNECT path logic. - match resolve_and_check_trusted_gateway(&host, port, gw, sandbox_entrypoint_pid).await { - Ok(addrs) => addrs, - Err(reason) => { - { - let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Other) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .http_request(HttpRequest::new( - method, - OcsfUrl::new("http", &host_lc, &path, port), - )) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .src_endpoint(Endpoint::from_ip(workload_addr.ip(), workload_addr.port())) - .actor_process( - Process::from_bypass(&binary_str, &pid_str, &ancestors_str) - .with_cmd_line(&cmdline_str), - ) - .firewall_rule(policy_str, "ssrf") - .message(format!( - "FORWARD blocked: trusted-gateway check failed for {host_lc}:{port}" - )) - .status_detail(&reason) - .build(); - ocsf_emit!(event); - } - emit_denial_simple( - denial_tx, - &host_lc, - port, - &binary_str, - &decision, - &reason, - "ssrf", - ); - emit_activity_simple(activity_tx, true, "ssrf"); - respond( - client, - &build_json_error_response( - 403, - "Forbidden", - "ssrf_denied", - &format!("{method} {host_lc}:{port} blocked: trusted-gateway check failed"), - ), - ) - .await?; - return Ok(()); - } - } - } else if !raw_allowed_ips.is_empty() { - // allowed_ips mode: validate resolved IPs against CIDR allowlist. - match parse_allowed_ips(&raw_allowed_ips) { - Ok(nets) => { - match resolve_and_check_allowed_ips(&host, port, &nets, sandbox_entrypoint_pid) - .await - { - Ok(addrs) => addrs, - Err(reason) => { - { - let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Other) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .http_request(HttpRequest::new( - method, - OcsfUrl::new("http", &host_lc, &path, port), - )) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .src_endpoint(Endpoint::from_ip( - workload_addr.ip(), - workload_addr.port(), - )) - .actor_process( - Process::from_bypass(&binary_str, &pid_str, &ancestors_str) - .with_cmd_line(&cmdline_str), - ) - .firewall_rule(policy_str, "ssrf") - .message(format!( - "FORWARD blocked: allowed_ips check failed for {host_lc}:{port}" - )) - .status_detail(&reason) - .build(); - ocsf_emit!(event); - } - emit_denial_simple( - denial_tx, - &host_lc, - port, - &binary_str, - &decision, - &reason, - "ssrf", - ); - emit_activity_simple(activity_tx, true, "ssrf"); - respond( - client, - &build_json_error_response( - 403, - "Forbidden", - "ssrf_denied", - &format!( - "{method} {host_lc}:{port} blocked: allowed_ips check failed" - ), - ), - ) - .await?; - return Ok(()); - } - } - } - Err(reason) => { - { - let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Other) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .http_request(HttpRequest::new( - method, - OcsfUrl::new("http", &host_lc, &path, port), - )) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .src_endpoint(Endpoint::from_ip(workload_addr.ip(), workload_addr.port())) - .actor_process( - Process::from_bypass(&binary_str, &pid_str, &ancestors_str) - .with_cmd_line(&cmdline_str), - ) - .firewall_rule(policy_str, "ssrf") - .message(format!( - "FORWARD blocked: invalid allowed_ips in policy for {host_lc}:{port}" - )) - .status_detail(&reason) - .build(); - ocsf_emit!(event); - } - emit_denial_simple( - denial_tx, - &host_lc, - port, - &binary_str, - &decision, - &reason, - "ssrf", - ); - emit_activity_simple(activity_tx, true, "ssrf"); - respond( - client, - &build_json_error_response( - 403, - "Forbidden", - "ssrf_denied", - &format!( - "{method} {host_lc}:{port} blocked: invalid allowed_ips in policy" - ), - ), - ) - .await?; - return Ok(()); - } - } - } else if exact_declared_endpoint_host { - // Exact declared hostname mode mirrors CONNECT: private resolved - // addresses are allowed for this operator-declared host:port, while - // always-blocked addresses and control-plane ports remain denied. - match resolve_and_check_declared_endpoint(&host, port, sandbox_entrypoint_pid).await { - Ok(addrs) => addrs, - Err(reason) => { - { - let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Other) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .http_request(HttpRequest::new( - method, - OcsfUrl::new("http", &host_lc, &path, port), - )) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .src_endpoint(Endpoint::from_ip(workload_addr.ip(), workload_addr.port())) - .actor_process( - Process::from_bypass(&binary_str, &pid_str, &ancestors_str) - .with_cmd_line(&cmdline_str), - ) - .firewall_rule(policy_str, "ssrf") - .message(format!( - "FORWARD blocked: declared endpoint check failed for {host_lc}:{port}" - )) - .status_detail(&reason) - .build(); - ocsf_emit!(event); - } - emit_denial_simple( - denial_tx, - &host_lc, - port, - &binary_str, - &decision, - &reason, - "ssrf", - ); - respond( - client, - &build_json_error_response( - 403, - "Forbidden", - "ssrf_denied", - &format!( - "{method} {host_lc}:{port} blocked: declared endpoint check failed" - ), - ), - ) - .await?; - return Ok(()); - } - } - } else { - // No allowed_ips: reject internal IPs, allow public IPs through. - match resolve_and_reject_internal(&host, port, sandbox_entrypoint_pid).await { - Ok(addrs) => addrs, - Err(reason) => { - { - let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Other) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .http_request(HttpRequest::new( - method, - OcsfUrl::new("http", &host_lc, &path, port), - )) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .src_endpoint(Endpoint::from_ip(workload_addr.ip(), workload_addr.port())) - .actor_process( - Process::from_bypass(&binary_str, &pid_str, &ancestors_str) - .with_cmd_line(&cmdline_str), - ) - .firewall_rule(policy_str, "ssrf") - .message(format!( - "FORWARD blocked: internal IP without allowed_ips for {host_lc}:{port}" - )) - .status_detail(&reason) - .build(); - ocsf_emit!(event); - } - emit_denial_simple( - denial_tx, - &host_lc, - port, - &binary_str, - &decision, - &reason, - "ssrf", - ); - emit_activity_simple(activity_tx, true, "ssrf"); - respond( - client, - &build_json_error_response( - 403, - "Forbidden", - "ssrf_denied", - &format!("{method} {host_lc}:{port} blocked: internal address"), - ), - ) - .await?; - return Ok(()); - } + Ok(connector) => connector, + Err(denial) => { + deny_forward_destination( + client, + &denial, + workload_addr, + method, + &host_lc, + port, + &path, + &binary_str, + &pid_str, + &ancestors_str, + &cmdline_str, + policy_str, + &decision, + denial_tx, + activity_tx, + ) + .await?; + return Ok(()); } }; @@ -4845,8 +4399,7 @@ async fn handle_forward_proxy( // directly: only TLS (CONNECT) tunnels chain through the corporate // proxy, since plain-HTTP forwarding would need absolute-form requests // rather than a CONNECT tunnel. - let dial_result = TcpStream::connect(addrs.as_slice()).await; - let mut upstream = match dial_result { + let mut upstream = match connector.connect().await { Ok(s) => s, Err(e) => { let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) @@ -5515,7 +5068,10 @@ network_policies: let engine = OpaEngine::from_strings_with_binary_identity_required(policy, data, false) .expect("relaxed engine"); - let decision = evaluate_endpoint_only_opa(&engine, "host.k3d.internal", 56123); + let decision = evaluate_endpoint_only_opa( + &engine, + EgressIntent::connect("host.k3d.internal".to_string(), 56123), + ); assert_eq!( decision.action, NetworkAction::Allow { @@ -5525,7 +5081,10 @@ network_policies: assert!(decision.binary.is_none()); assert!(decision.ancestors.is_empty()); - let denied = evaluate_endpoint_only_opa(&engine, "api.example.com", 443); + let denied = evaluate_endpoint_only_opa( + &engine, + EgressIntent::connect("api.example.com".to_string(), 443), + ); assert!( matches!(denied.action, NetworkAction::Deny { .. }), "endpoint-only mode must still deny undeclared endpoints" @@ -6125,11 +5684,14 @@ network_policies: ) { let policy = include_str!("../data/sandbox-policy.rego"); let engine = OpaEngine::from_strings(policy, data).unwrap(); - let decision = ConnectDecision { + let decision = EgressDecision { + intent: EgressIntent::forward_http(host.to_string(), port), action: NetworkAction::Allow { matched_policy: Some(policy_name.to_string()), }, generation: engine.current_generation(), + identity: ProcessIdentityEvidence::Available, + endpoint: EndpointDecision::default(), binary: Some(PathBuf::from("/usr/bin/node")), binary_pid: None, ancestors: vec![], @@ -9719,6 +9281,66 @@ network_policies: (completed, stdout, denial_stages) } + /// Drives an absolute-form request through the same explicit-proxy entry + /// point used by CONNECT and returns the response and denial stages. + async fn drive_forward_through_handler( + endpoint_yaml: &str, + target: &str, + ) -> (Vec, Vec) { + const POLICY_REGO: &str = include_str!("../data/sandbox-policy.rego"); + + let exe = std::env::current_exe().expect("current_exe"); + let data = format!( + r#"network_policies: + test_allow: + name: test_allow + endpoints: +{endpoint_yaml} binaries: + - {{ path: "{exe}" }} +"#, + exe = exe.display(), + ); + let engine = Arc::new(OpaEngine::from_strings(POLICY_REGO, &data).expect("load policy")); + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let proxy_port = listener.local_addr().unwrap().port(); + let target = target.to_string(); + let client = tokio::spawn(async move { + let mut socket = TcpStream::connect(("127.0.0.1", proxy_port)).await.unwrap(); + let request = format!("GET {target} HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n"); + socket.write_all(request.as_bytes()).await.unwrap(); + let mut response = Vec::new(); + socket.read_to_end(&mut response).await.unwrap(); + response + }); + + let (server, _) = listener.accept().await.unwrap(); + let (denial_tx, mut denial_rx) = mpsc::unbounded_channel(); + Box::pin(handle_tcp_connection( + server, + engine, + Arc::new(BinaryIdentityCache::new()), + Arc::new(AtomicU32::new(std::process::id())), + None, + None, + None, + Arc::new(None), + None, + None, + Some(denial_tx), + None, + )) + .await + .expect("forward handler should complete"); + + let response = client.await.expect("client task"); + let mut denial_stages = Vec::new(); + while let Ok(event) = denial_rx.try_recv() { + denial_stages.push(event.denial_stage); + } + (response, denial_stages) + } + /// End-to-end regression for the gator finding on PR #2162: with no TLS /// termination state, a terminating `CONNECT` must have its 503 written as /// the FIRST bytes on the socket — never after a `200 Connection @@ -9790,6 +9412,29 @@ network_policies: ); } + #[tokio::test] + async fn forward_handler_preserves_ssrf_response_and_denial_stage() { + if !cfg!(target_os = "linux") { + eprintln!("skipping: handler identity binding requires /proc (Linux)"); + return; + } + + let (response, denial_stages) = Box::pin(drive_forward_through_handler( + " - { host: \"127.0.0.1\", port: 80 }\n", + "http://127.0.0.1/private", + )) + .await; + + let response = String::from_utf8_lossy(&response); + assert!( + response.starts_with("HTTP/1.1 403 Forbidden"), + "internal forward destination must get the SSRF 403; got: {response:?}" + ); + assert!(response.contains("ssrf_denied")); + assert!(response.contains("GET 127.0.0.1:80 blocked: allowed_ips check failed")); + assert_eq!(denial_stages, ["ssrf"]); + } + /// A real `tls: skip` policy path through the handler is exempt from the /// fail-closed gate even with no TLS termination state: the handler proceeds /// past the refusal to the raw-tunnel upstream connect (which stalls on the @@ -9868,9 +9513,12 @@ network_policies: panic!("glob binary must be allowed, got deny: {reason}") } } - let decision = ConnectDecision { + let decision = EgressDecision { + intent: EgressIntent::connect("203.0.113.10".to_string(), 443), action, generation, + identity: ProcessIdentityEvidence::Available, + endpoint: EndpointDecision::default(), binary: Some(input.binary_path), binary_pid: Some(1), ancestors: vec![], diff --git a/crates/openshell-supervisor-network/src/proxy/destination.rs b/crates/openshell-supervisor-network/src/proxy/destination.rs new file mode 100644 index 0000000000..b9fc42cf75 --- /dev/null +++ b/crates/openshell-supervisor-network/src/proxy/destination.rs @@ -0,0 +1,197 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Shared external destination validation and upstream dial boundary. + +use super::{ + implicit_allowed_ips_for_ip_host, is_host_gateway_alias, parse_allowed_ips, + resolve_and_check_allowed_ips, resolve_and_check_declared_endpoint, + resolve_and_check_trusted_gateway, resolve_and_reject_internal, +}; +use std::net::{IpAddr, SocketAddr}; +use tokio::net::TcpStream; + +/// Inputs needed to apply the current SSRF and endpoint destination policy. +pub(super) struct DestinationRequest<'a> { + pub(super) host: &'a str, + pub(super) normalized_host: &'a str, + pub(super) port: u16, + pub(super) sandbox_entrypoint_pid: u32, + pub(super) trusted_host_gateway: Option, + pub(super) raw_allowed_ips: Vec, + pub(super) exact_declared_endpoint_host: bool, +} + +/// Destination-validation branch that rejected an egress request. +/// +/// Adapters use this classification to preserve their existing HTTP response +/// and OCSF message shapes while sharing the underlying validation logic. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum DestinationDenialKind { + TrustedGateway, + InvalidAllowedIps, + AllowedIps, + DeclaredEndpoint, + InternalAddress, +} + +#[derive(Debug)] +pub(super) struct DestinationDenial { + pub(super) kind: DestinationDenialKind, + pub(super) reason: String, +} + +impl DestinationDenial { + fn new(kind: DestinationDenialKind, reason: String) -> Self { + Self { kind, reason } + } +} + +/// Validated, but not yet opened, upstream destination. +/// +/// The explicit proxy adapter controls when `connect` is called so CONNECT and +/// forward HTTP retain their current upstream-dial timing during the refactor. +pub(super) struct UpstreamConnector { + host: String, + port: u16, + addrs: Vec, +} + +impl UpstreamConnector { + pub(super) fn addrs(&self) -> &[SocketAddr] { + &self.addrs + } + + pub(super) async fn connect(&self) -> std::io::Result { + tracing::debug!( + host = %self.host, + port = self.port, + address_count = self.addrs.len(), + "Opening validated upstream connection" + ); + TcpStream::connect(self.addrs.as_slice()).await + } + + fn new(host: &str, port: u16, addrs: Vec) -> Self { + Self { + host: host.to_string(), + port, + addrs, + } + } +} + +/// Resolve and validate a destination using the existing proxy security rules. +pub(super) async fn validate_destination( + request: DestinationRequest<'_>, +) -> Result { + let DestinationRequest { + host, + normalized_host, + port, + sandbox_entrypoint_pid, + trusted_host_gateway, + mut raw_allowed_ips, + exact_declared_endpoint_host, + } = request; + + if raw_allowed_ips.is_empty() { + raw_allowed_ips = implicit_allowed_ips_for_ip_host(host); + } + + #[allow(clippy::if_not_else)] + let addrs = if is_host_gateway_alias(normalized_host) + && let Some(gateway) = trusted_host_gateway + { + resolve_and_check_trusted_gateway(host, port, gateway, sandbox_entrypoint_pid) + .await + .map_err(|reason| { + DestinationDenial::new(DestinationDenialKind::TrustedGateway, reason) + })? + } else if !raw_allowed_ips.is_empty() { + let networks = parse_allowed_ips(&raw_allowed_ips).map_err(|reason| { + DestinationDenial::new(DestinationDenialKind::InvalidAllowedIps, reason) + })?; + resolve_and_check_allowed_ips(host, port, &networks, sandbox_entrypoint_pid) + .await + .map_err(|reason| DestinationDenial::new(DestinationDenialKind::AllowedIps, reason))? + } else if exact_declared_endpoint_host { + resolve_and_check_declared_endpoint(host, port, sandbox_entrypoint_pid) + .await + .map_err(|reason| { + DestinationDenial::new(DestinationDenialKind::DeclaredEndpoint, reason) + })? + } else { + resolve_and_reject_internal(host, port, sandbox_entrypoint_pid) + .await + .map_err(|reason| { + DestinationDenial::new(DestinationDenialKind::InternalAddress, reason) + })? + }; + + Ok(UpstreamConnector::new(host, port, addrs)) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::net::{IpAddr, Ipv4Addr}; + + fn request(host: &str) -> DestinationRequest<'_> { + DestinationRequest { + host, + normalized_host: host, + port: 80, + sandbox_entrypoint_pid: 0, + trusted_host_gateway: None, + raw_allowed_ips: vec![], + exact_declared_endpoint_host: false, + } + } + + #[tokio::test] + async fn default_mode_classifies_loopback_as_internal_address() { + let denial = validate_destination(request("127.0.0.1")) + .await + .err() + .expect("loopback must be denied"); + + assert_eq!(denial.kind, DestinationDenialKind::InternalAddress); + } + + #[tokio::test] + async fn invalid_allowed_ips_has_a_distinct_denial_kind() { + let mut request = request("api.example.test"); + request.raw_allowed_ips = vec!["not-an-ip".to_string()]; + let denial = validate_destination(request) + .await + .err() + .expect("invalid allowed_ips must be denied"); + + assert_eq!(denial.kind, DestinationDenialKind::InvalidAllowedIps); + } + + #[tokio::test] + async fn declared_endpoint_preserves_its_denial_classification() { + let mut request = request("127.0.0.1"); + request.exact_declared_endpoint_host = true; + let denial = validate_destination(request) + .await + .err() + .expect("declared loopback must remain denied"); + + assert_eq!(denial.kind, DestinationDenialKind::DeclaredEndpoint); + } + + #[tokio::test] + async fn trusted_gateway_preserves_its_denial_classification() { + let mut request = request("host.openshell.internal"); + request.trusted_host_gateway = Some(IpAddr::V4(Ipv4Addr::LOCALHOST)); + let denial = validate_destination(request) + .await + .err() + .expect("loopback cannot be a trusted gateway"); + + assert_eq!(denial.kind, DestinationDenialKind::TrustedGateway); + } +} diff --git a/crates/openshell-supervisor-network/src/proxy/egress.rs b/crates/openshell-supervisor-network/src/proxy/egress.rs new file mode 100644 index 0000000000..3d57dbe4ec --- /dev/null +++ b/crates/openshell-supervisor-network/src/proxy/egress.rs @@ -0,0 +1,144 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Transport-neutral egress inputs and authorization results. +//! +//! Explicit proxy adapters normalize their protocol-specific request into an +//! [`EgressIntent`]. Authorization then returns an [`EgressDecision`] that is +//! consumed by destination validation and relay selection. Keeping these types +//! independent of CONNECT and forward HTTP prevents policy behavior from +//! drifting as more adapters are added. + +use crate::opa::NetworkAction; +use std::path::PathBuf; + +#[derive(Debug, Clone)] +pub(super) struct L7ConfigSnapshot { + pub(super) config: crate::l7::L7EndpointConfig, +} + +#[derive(Debug, Clone)] +pub(super) struct L7RouteSnapshot { + pub(super) configs: Vec, + pub(super) generation: u64, +} + +/// Endpoint metadata materialized for an allowed egress decision. +/// +/// The migration hydrates these fields at the same points the legacy handlers +/// queried them so policy-reload and upstream-connect timing remain unchanged. +#[derive(Debug, Clone)] +pub(super) struct EndpointDecision { + pub(super) tls_mode: crate::l7::TlsMode, + pub(super) l7_route: Option, + pub(super) raw_allowed_ips: Vec, + pub(super) exact_declared_host: bool, +} + +impl Default for EndpointDecision { + fn default() -> Self { + Self { + tls_mode: crate::l7::TlsMode::Auto, + l7_route: None, + raw_allowed_ips: vec![], + exact_declared_host: false, + } + } +} + +/// Userland surface through which an external egress request arrived. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) enum EgressTransport { + Connect, + ForwardHttp, +} + +/// Destination requested by an explicit proxy adapter. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct RequestedDestination { + pub(super) host: String, + pub(super) port: u16, +} + +/// Transport-neutral description of an external egress request. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct EgressIntent { + pub(super) transport: EgressTransport, + pub(super) destination: RequestedDestination, +} + +impl EgressIntent { + pub(super) fn connect(host: String, port: u16) -> Self { + Self::new(EgressTransport::Connect, host, port) + } + + pub(super) fn forward_http(host: String, port: u16) -> Self { + Self::new(EgressTransport::ForwardHttp, host, port) + } + + fn new(transport: EgressTransport, host: String, port: u16) -> Self { + Self { + transport, + destination: RequestedDestination { host, port }, + } + } +} + +/// Why process identity is absent from an egress decision. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] +pub(super) enum IdentityUnavailableReason { + EndpointOnlyMode, + LookupFailed, + UnsupportedPlatform, +} + +/// Process evidence captured for policy evaluation and audit logging. +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] +pub(super) enum ProcessIdentityEvidence { + Available, + Unavailable(IdentityUnavailableReason), +} + +/// Result of authorizing a normalized egress intent. +/// +/// The identity fields intentionally mirror the former CONNECT-specific +/// decision during the compatibility migration. Endpoint configuration is +/// hydrated at the legacy query points without changing lookup precedence or +/// failure defaults. +pub(super) struct EgressDecision { + pub(super) intent: EgressIntent, + pub(super) action: NetworkAction, + /// Policy generation used for the L4 network decision. + pub(super) generation: u64, + /// Whether process identity evidence was available to policy evaluation. + pub(super) identity: ProcessIdentityEvidence, + /// Endpoint behavior hydrated for destination validation and relays. + pub(super) endpoint: EndpointDecision, + /// Resolved binary path. + pub(super) binary: Option, + /// PID owning the socket. + pub(super) binary_pid: Option, + /// Ancestor binary paths from process tree walk. + pub(super) ancestors: Vec, + /// Cmdline-derived absolute paths (for script detection). + pub(super) cmdline_paths: Vec, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn adapters_create_transport_specific_intents() { + let connect = EgressIntent::connect("api.example.com".to_string(), 443); + let forward = EgressIntent::forward_http("api.example.com".to_string(), 80); + + assert_eq!(connect.transport, EgressTransport::Connect); + assert_eq!(connect.destination.host, "api.example.com"); + assert_eq!(connect.destination.port, 443); + assert_eq!(forward.transport, EgressTransport::ForwardHttp); + assert_eq!(forward.destination.port, 80); + } +} diff --git a/crates/openshell-supervisor-network/src/proxy/relay.rs b/crates/openshell-supervisor-network/src/proxy/relay.rs new file mode 100644 index 0000000000..68de77dcfd --- /dev/null +++ b/crates/openshell-supervisor-network/src/proxy/relay.rs @@ -0,0 +1,149 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Shared relay primitives for authorized explicit-proxy egress. + +use super::{EgressDecision, L7RouteSnapshot, emit_l7_tunnel_close_after_policy_change}; +use crate::l7::relay::L7EvalContext; +use crate::opa::{NetworkAction, OpaEngine}; +use miette::{IntoDiagnostic, Result}; +use openshell_core::activity::ActivitySender; +use openshell_core::proto::ProviderProfileCredential; +use openshell_core::secrets::SecretResolver; +use std::collections::HashMap; +use std::sync::Arc; +use tokio::io::{AsyncRead, AsyncWrite}; + +type DynamicCredentials = Arc>>; + +/// Build the request-processing context shared by CONNECT and forward HTTP. +pub(super) fn http_context( + decision: &EgressDecision, + secret_resolver: Option>, + activity_tx: Option, + dynamic_credentials: Option, + agent_proposals: openshell_core::proposals::AgentProposals, +) -> L7EvalContext { + let policy_name = match &decision.action { + NetworkAction::Allow { matched_policy } => matched_policy.clone().unwrap_or_default(), + NetworkAction::Deny { .. } => String::new(), + }; + + L7EvalContext { + host: decision.intent.destination.host.clone(), + port: decision.intent.destination.port, + policy_name, + binary_path: decision + .binary + .as_ref() + .map(|path| path.to_string_lossy().into_owned()) + .unwrap_or_default(), + ancestors: decision + .ancestors + .iter() + .map(|path| path.to_string_lossy().into_owned()) + .collect(), + cmdline_paths: decision + .cmdline_paths + .iter() + .map(|path| path.to_string_lossy().into_owned()) + .collect(), + secret_resolver, + activity_tx, + dynamic_credentials: dynamic_credentials.clone(), + token_grant_resolver: dynamic_credentials + .as_ref() + .map(|_| crate::l7::token_grant_injection::default_resolver()), + agent_proposals, + } +} + +/// Relay an HTTP/1 stream using the endpoint's current L7 configuration. +/// +/// CONNECT plaintext and TLS-terminated streams both enter through this +/// function. Forward HTTP will provide a buffered first request to the same +/// boundary in the next migration step. +pub(super) async fn relay_http_stream( + route: Option<&L7RouteSnapshot>, + opa_engine: &Arc, + decision: &EgressDecision, + client: &mut C, + upstream: &mut U, + context: &L7EvalContext, +) -> Result<()> +where + C: AsyncRead + AsyncWrite + Unpin + Send, + U: AsyncRead + AsyncWrite + Unpin + Send, +{ + if let Some(route) = route.filter(|route| !route.configs.is_empty()) { + let tunnel_engine = match opa_engine.clone_engine_for_tunnel(route.generation) { + Ok(engine) => engine, + Err(error) => { + emit_l7_tunnel_close_after_policy_change( + &decision.intent.destination.host, + decision.intent.destination.port, + error, + ); + return Ok(()); + } + }; + + if route.configs.len() == 1 { + crate::l7::relay::relay_with_inspection( + &route.configs[0].config, + tunnel_engine, + client, + upstream, + context, + ) + .await + } else { + let configs = route + .configs + .iter() + .map(|snapshot| snapshot.config.clone()) + .collect::>(); + crate::l7::relay::relay_with_route_selection( + &configs, + tunnel_engine, + client, + upstream, + context, + ) + .await + } + } else { + let generation = route.map_or(decision.generation, |route| route.generation); + let generation_guard = match opa_engine.generation_guard(generation) { + Ok(guard) => guard, + Err(error) => { + emit_l7_tunnel_close_after_policy_change( + &decision.intent.destination.host, + decision.intent.destination.port, + error, + ); + return Ok(()); + } + }; + crate::l7::relay::relay_passthrough_with_credentials( + client, + upstream, + context, + &generation_guard, + Some(opa_engine), + ) + .await + } +} + +/// Relay a policy-authorized raw TCP stream. +pub(super) async fn relay_tcp(client: &mut C, upstream: &mut U) -> Result<()> +where + C: AsyncRead + AsyncWrite + Unpin, + U: AsyncRead + AsyncWrite + Unpin, +{ + tokio::io::copy_bidirectional(client, upstream) + .await + .into_diagnostic()?; + Ok(()) +} From 49de24295efb7e3ec9b84eed9abb3386adb9fdf5 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Tue, 14 Jul 2026 14:31:37 -0700 Subject: [PATCH 02/30] test(network): cover shared proxy egress paths Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- e2e/rust/Cargo.toml | 5 + e2e/rust/tests/proxy_egress_pipeline.rs | 847 ++++++++++++++++++++++++ 2 files changed, 852 insertions(+) create mode 100644 e2e/rust/tests/proxy_egress_pipeline.rs diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 96e451b3a1..c8ef57f693 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -123,6 +123,11 @@ name = "workspace_lifecycle" path = "tests/workspace_lifecycle.rs" required-features = ["e2e"] +[[test]] +name = "proxy_egress_pipeline" +path = "tests/proxy_egress_pipeline.rs" +required-features = ["e2e-host-gateway"] + [[test]] name = "gpu" path = "tests/gpu.rs" diff --git a/e2e/rust/tests/proxy_egress_pipeline.rs b/e2e/rust/tests/proxy_egress_pipeline.rs new file mode 100644 index 0000000000..f226f1dd13 --- /dev/null +++ b/e2e/rust/tests/proxy_egress_pipeline.rs @@ -0,0 +1,847 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#![cfg(feature = "e2e")] + +//! E2E coverage for the shared explicit-proxy egress pipeline. +//! +//! These tests exercise behavior that must remain identical while CONNECT and +//! forward HTTP converge on shared authorization, destination, and relay +//! primitives: +//! - live policy reloads affect new requests through both adapters and close a +//! pre-existing CONNECT HTTP stream before its next request is forwarded; +//! - `tls: skip` selects a byte-transparent TCP relay; +//! - provider placeholders in HTTP headers and opted-in REST bodies are +//! resolved through both adapters without appearing in test output. + +use std::io::{self, Error, ErrorKind, Write}; +use std::process::Stdio; +use std::sync::Mutex; + +use openshell_e2e::harness::binary::openshell_cmd; +use openshell_e2e::harness::sandbox::SandboxGuard; +use serde_json::Value; +use tempfile::NamedTempFile; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::task::JoinHandle; + +const TEST_SERVER_HOST: &str = "host.openshell.internal"; +const PROVIDER_NAME: &str = "e2e-proxy-egress-credentials"; +const TOKEN_ENV: &str = "PROXY_E2E_TOKEN"; +const TEST_SECRET: &str = "sk-e2e-proxy-egress-secret"; +const PLACEHOLDER_PREFIX: &str = "openshell:resolve:env:"; +const PRIVATE_ALLOWED_IPS: &str = r#" allowed_ips: + - "10.0.0.0/8" + - "172.0.0.0/8" + - "192.168.0.0/16" + - "fc00::/7""#; +static PROVIDER_LOCK: Mutex<()> = Mutex::new(()); + +async fn run_cli(args: &[&str]) -> Result { + let mut cmd = openshell_cmd(); + cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); + + let output = cmd + .output() + .await + .map_err(|error| format!("failed to spawn openshell {}: {error}", args.join(" ")))?; + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + let combined = format!("{stdout}{stderr}"); + + if !output.status.success() { + return Err(format!( + "openshell {} failed (exit {:?}):\n{combined}", + args.join(" "), + output.status.code() + )); + } + + Ok(combined) +} + +async fn delete_provider(name: &str) { + let mut cmd = openshell_cmd(); + cmd.args(["provider", "delete", name]) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + let _ = cmd.status().await; +} + +async fn create_generic_provider(name: &str) -> Result { + let credential = format!("{TOKEN_ENV}={TEST_SECRET}"); + run_cli(&[ + "provider", + "create", + "--name", + name, + "--type", + "generic", + "--credential", + &credential, + ]) + .await +} + +fn write_policy(host: &str, port: u16, endpoint_options: &str) -> Result { + let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; + let policy = format!( + r#"version: 1 + +filesystem_policy: + include_workdir: true + read_only: + - /usr + - /lib + - /proc + - /dev/urandom + - /app + - /etc + - /var/log + read_write: + - /sandbox + - /tmp + - /dev/null + +landlock: + compatibility: best_effort + +process: + run_as_user: sandbox + run_as_group: sandbox + +network_policies: + proxy_egress_test: + name: proxy_egress_test + endpoints: + - host: {host} + port: {port} +{endpoint_options} +{PRIVATE_ALLOWED_IPS} + binaries: + - path: "/**" +"# + ); + file.write_all(policy.as_bytes()) + .map_err(|error| format!("write policy: {error}"))?; + file.flush() + .map_err(|error| format!("flush policy: {error}"))?; + Ok(file) +} + +fn write_denied_policy() -> Result { + let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; + let policy = r#"version: 1 + +filesystem_policy: + include_workdir: true + read_only: + - /usr + - /lib + - /proc + - /dev/urandom + - /app + - /etc + - /var/log + read_write: + - /sandbox + - /tmp + - /dev/null + +landlock: + compatibility: best_effort + +process: + run_as_user: sandbox + run_as_group: sandbox + +network_policies: {} +"#; + file.write_all(policy.as_bytes()) + .map_err(|error| format!("write policy: {error}"))?; + file.flush() + .map_err(|error| format!("flush policy: {error}"))?; + Ok(file) +} + +fn policy_path(file: &NamedTempFile) -> String { + file.path() + .to_str() + .expect("temporary policy path should be utf-8") + .to_string() +} + +async fn read_until(stream: &mut TcpStream, marker: &[u8]) -> io::Result> { + let mut data = Vec::new(); + let mut buffer = [0_u8; 4096]; + loop { + let read = stream.read(&mut buffer).await?; + if read == 0 { + return Ok(data); + } + data.extend_from_slice(&buffer[..read]); + if data.windows(marker.len()).any(|window| window == marker) { + return Ok(data); + } + } +} + +fn header_end(bytes: &[u8]) -> Option { + bytes + .windows(4) + .position(|window| window == b"\r\n\r\n") + .map(|position| position + 4) +} + +fn content_length(headers: &[u8]) -> io::Result { + let text = + std::str::from_utf8(headers).map_err(|error| Error::new(ErrorKind::InvalidData, error))?; + Ok(text + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.trim() + .eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().ok()) + .flatten() + }) + .unwrap_or(0)) +} + +async fn read_http_request(stream: &mut TcpStream) -> io::Result>> { + let mut request = read_until(stream, b"\r\n\r\n").await?; + if request.is_empty() { + return Ok(None); + } + let headers_end = header_end(&request) + .ok_or_else(|| Error::new(ErrorKind::UnexpectedEof, "incomplete HTTP headers"))?; + let body_length = content_length(&request[..headers_end])?; + let total_length = headers_end + body_length; + while request.len() < total_length { + let mut buffer = vec![0_u8; total_length - request.len()]; + let read = stream.read(&mut buffer).await?; + if read == 0 { + return Err(Error::new(ErrorKind::UnexpectedEof, "incomplete HTTP body")); + } + request.extend_from_slice(&buffer[..read]); + } + request.truncate(total_length); + Ok(Some(request)) +} + +struct KeepAliveHttpServer { + port: u16, + task: JoinHandle<()>, +} + +impl KeepAliveHttpServer { + async fn start() -> Result { + let listener = TcpListener::bind(("0.0.0.0", 0)) + .await + .map_err(|error| format!("bind HTTP server: {error}"))?; + let port = listener + .local_addr() + .map_err(|error| format!("read HTTP server address: {error}"))? + .port(); + let task = tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + tokio::spawn(async move { + let _ = handle_keep_alive_connection(stream).await; + }); + } + }); + Ok(Self { port, task }) + } +} + +impl Drop for KeepAliveHttpServer { + fn drop(&mut self) { + self.task.abort(); + } +} + +async fn handle_keep_alive_connection(mut stream: TcpStream) -> io::Result<()> { + while let Some(request) = read_http_request(&mut stream).await? { + let close = String::from_utf8_lossy(&request) + .lines() + .any(|line| line.eq_ignore_ascii_case("connection: close")); + let connection = if close { "close" } else { "keep-alive" }; + let response = + format!("HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: {connection}\r\n\r\nok"); + stream.write_all(response.as_bytes()).await?; + if close { + return Ok(()); + } + } + Ok(()) +} + +struct EchoServer { + port: u16, + task: JoinHandle<()>, +} + +impl EchoServer { + async fn start() -> Result { + let listener = TcpListener::bind(("0.0.0.0", 0)) + .await + .map_err(|error| format!("bind echo server: {error}"))?; + let port = listener + .local_addr() + .map_err(|error| format!("read echo server address: {error}"))? + .port(); + let task = tokio::spawn(async move { + while let Ok((mut stream, _)) = listener.accept().await { + tokio::spawn(async move { + let mut buffer = [0_u8; 4096]; + loop { + let Ok(read) = stream.read(&mut buffer).await else { + break; + }; + if read == 0 || stream.write_all(&buffer[..read]).await.is_err() { + break; + } + } + }); + } + }); + Ok(Self { port, task }) + } +} + +impl Drop for EchoServer { + fn drop(&mut self) { + self.task.abort(); + } +} + +struct CredentialProbeServer { + port: u16, + task: JoinHandle<()>, +} + +impl CredentialProbeServer { + async fn start() -> Result { + let listener = TcpListener::bind(("0.0.0.0", 0)) + .await + .map_err(|error| format!("bind credential probe: {error}"))?; + let port = listener + .local_addr() + .map_err(|error| format!("read credential probe address: {error}"))? + .port(); + let task = tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + tokio::spawn(async move { + let _ = handle_credential_probe(stream).await; + }); + } + }); + Ok(Self { port, task }) + } +} + +impl Drop for CredentialProbeServer { + fn drop(&mut self) { + self.task.abort(); + } +} + +async fn handle_credential_probe(mut stream: TcpStream) -> io::Result<()> { + let request = read_http_request(&mut stream) + .await? + .ok_or_else(|| Error::new(ErrorKind::UnexpectedEof, "missing HTTP request"))?; + let headers_end = header_end(&request) + .ok_or_else(|| Error::new(ErrorKind::UnexpectedEof, "incomplete HTTP headers"))?; + let headers = String::from_utf8_lossy(&request[..headers_end]); + let expected_authorization = format!("Bearer {TEST_SECRET}"); + let header_resolved = headers.lines().any(|line| { + line.split_once(':').is_some_and(|(name, value)| { + name.eq_ignore_ascii_case("authorization") && value.trim() == expected_authorization + }) + }); + let body_resolved = request[headers_end..] + .windows(TEST_SECRET.len()) + .any(|window| window == TEST_SECRET.as_bytes()); + let saw_placeholder = request + .windows(PLACEHOLDER_PREFIX.len()) + .any(|window| window == PLACEHOLDER_PREFIX.as_bytes()); + let body = serde_json::json!({ + "body_resolved": body_resolved, + "header_resolved": header_resolved, + "saw_placeholder": saw_placeholder, + }) + .to_string(); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + stream.write_all(response.as_bytes()).await +} + +fn proxy_status_script(host: &str, port: u16) -> String { + format!( + r#" +import json +import os +import socket +import urllib.parse + +HOST = {host:?} +PORT = {port} + +def proxy_parts(): + proxy_url = next( + os.environ[name] + for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") + if os.environ.get(name) + ) + parsed = urllib.parse.urlparse(proxy_url) + return parsed.hostname, parsed.port or 80 + +def read_headers(sock): + data = b"" + while b"\r\n\r\n" not in data: + chunk = sock.recv(4096) + if not chunk: + break + data += chunk + return data + +def status(response): + parts = response.split(None, 2) + return int(parts[1]) if len(parts) > 1 else 0 + +def forward_status(): + proxy_host, proxy_port = proxy_parts() + target = f"{{HOST}}:{{PORT}}" + with socket.create_connection((proxy_host, proxy_port), timeout=10) as sock: + sock.sendall( + f"GET http://{{target}}/forward HTTP/1.1\r\n" + f"Host: {{target}}\r\nConnection: close\r\n\r\n".encode() + ) + return status(read_headers(sock)) + +def connect_status(): + proxy_host, proxy_port = proxy_parts() + target = f"{{HOST}}:{{PORT}}" + with socket.create_connection((proxy_host, proxy_port), timeout=10) as sock: + sock.sendall(f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode()) + code = status(read_headers(sock)) + if code != 200: + return code + sock.sendall( + f"GET /connect HTTP/1.1\r\nHost: {{target}}\r\nConnection: close\r\n\r\n".encode() + ) + return status(read_headers(sock)) + +print(json.dumps({{"connect": connect_status(), "forward": forward_status()}}, sort_keys=True)) +"#, + host = host, + port = port, + ) +} + +fn persistent_connect_script(host: &str, port: u16) -> String { + format!( + r#" +import json +import os +import socket +import time +import urllib.parse + +HOST = {host:?} +PORT = {port} +READY = "/tmp/proxy-reload-ready" +GO = "/tmp/proxy-reload-go" +RESULT = "/tmp/proxy-reload-result" + +def proxy_parts(): + proxy_url = next( + os.environ[name] + for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") + if os.environ.get(name) + ) + parsed = urllib.parse.urlparse(proxy_url) + return parsed.hostname, parsed.port or 80 + +def read_response(sock): + data = b"" + while b"\r\n\r\n" not in data: + chunk = sock.recv(4096) + if not chunk: + return 0 + data += chunk + headers, body = data.split(b"\r\n\r\n", 1) + length = 0 + for line in headers.split(b"\r\n")[1:]: + if line.lower().startswith(b"content-length:"): + length = int(line.split(b":", 1)[1].strip()) + while len(body) < length: + chunk = sock.recv(4096) + if not chunk: + return 0 + body += chunk + return int(headers.split(None, 2)[1]) + +proxy_host, proxy_port = proxy_parts() +target = f"{{HOST}}:{{PORT}}" +failed_closed = False +second_status = 0 +try: + with socket.create_connection((proxy_host, proxy_port), timeout=10) as sock: + sock.sendall(f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode()) + if read_response(sock) != 200: + raise RuntimeError("initial CONNECT was denied") + sock.sendall( + f"GET /before-reload HTTP/1.1\r\nHost: {{target}}\r\nConnection: keep-alive\r\n\r\n".encode() + ) + if read_response(sock) != 200: + raise RuntimeError("initial tunneled request was denied") + open(READY, "w").close() + deadline = time.monotonic() + 120 + while not os.path.exists(GO) and time.monotonic() < deadline: + time.sleep(0.1) + if not os.path.exists(GO): + raise RuntimeError("timed out waiting for policy reload signal") + try: + sock.sendall( + f"GET /after-reload HTTP/1.1\r\nHost: {{target}}\r\nConnection: close\r\n\r\n".encode() + ) + second_status = read_response(sock) + except OSError: + second_status = 0 + failed_closed = second_status != 200 +finally: + with open(RESULT, "w") as result: + json.dump({{"failed_closed": failed_closed, "second_status": second_status}}, result, sort_keys=True) +"#, + host = host, + port = port, + ) +} + +async fn wait_for_sandbox_file(guard: &SandboxGuard, path: &str, log_path: &str) -> String { + let script = format!( + r#"import os, time +deadline = time.monotonic() + 60 +while not os.path.exists({path:?}) and time.monotonic() < deadline: + time.sleep(0.1) +if not os.path.exists({path:?}): + if os.path.exists({log_path:?}): + print(open({log_path:?}).read()) + raise SystemExit("timed out waiting for {path}") +print(open({path:?}).read()) +"# + ); + guard + .exec(&["python3", "-c", &script]) + .await + .unwrap_or_else(|error| panic!("wait for sandbox file {path}: {error}")) +} + +fn parse_json_line(output: &str) -> Value { + output + .lines() + .filter_map(|line| serde_json::from_str::(line.trim()).ok()) + .next_back() + .unwrap_or_else(|| panic!("missing JSON result in sandbox output:\n{output}")) +} + +#[tokio::test] +async fn policy_reload_updates_both_adapters_and_closes_existing_http_tunnel() { + let server = KeepAliveHttpServer::start() + .await + .expect("start keep-alive HTTP server"); + let policy_a = write_policy(TEST_SERVER_HOST, server.port, "").expect("write policy A"); + let policy_b = write_denied_policy().expect("write policy B"); + let policy_a_path = policy_path(&policy_a); + let policy_b_path = policy_path(&policy_b); + + let mut guard = SandboxGuard::create_keep_with_args( + &["--policy", &policy_a_path], + &["sh", "-c", "echo Ready; sleep infinity"], + "Ready", + ) + .await + .expect("create keep sandbox"); + + run_cli(&[ + "policy", + "set", + &guard.name, + "--policy", + &policy_a_path, + "--wait", + "--timeout", + "120", + ]) + .await + .expect("wait for policy A"); + + let persistent_script = persistent_connect_script(TEST_SERVER_HOST, server.port); + guard + .exec(&[ + "sh", + "-c", + "nohup python3 -c \"$1\" >/tmp/proxy-reload-client.log 2>&1 &", + "proxy-reload-client", + &persistent_script, + ]) + .await + .expect("start persistent CONNECT client"); + wait_for_sandbox_file( + &guard, + "/tmp/proxy-reload-ready", + "/tmp/proxy-reload-client.log", + ) + .await; + + let status_script = proxy_status_script(TEST_SERVER_HOST, server.port); + let before = guard + .exec(&["python3", "-c", &status_script]) + .await + .expect("exercise both adapters before reload"); + let before = parse_json_line(&before); + assert_eq!(before["connect"], 200, "CONNECT before reload: {before}"); + assert_eq!( + before["forward"], 200, + "forward HTTP before reload: {before}" + ); + + run_cli(&[ + "policy", + "set", + &guard.name, + "--policy", + &policy_b_path, + "--wait", + "--timeout", + "120", + ]) + .await + .expect("publish and wait for policy B"); + + guard + .exec(&["sh", "-c", "touch /tmp/proxy-reload-go"]) + .await + .expect("release persistent CONNECT client"); + let stale_tunnel = wait_for_sandbox_file( + &guard, + "/tmp/proxy-reload-result", + "/tmp/proxy-reload-client.log", + ) + .await; + let stale_tunnel = parse_json_line(&stale_tunnel); + assert_eq!( + stale_tunnel["failed_closed"], true, + "existing CONNECT HTTP stream forwarded after policy reload: {stale_tunnel}" + ); + + let after = guard + .exec(&["python3", "-c", &status_script]) + .await + .expect("exercise both adapters after reload"); + let after = parse_json_line(&after); + assert_eq!(after["connect"], 403, "CONNECT after reload: {after}"); + assert_eq!(after["forward"], 403, "forward HTTP after reload: {after}"); + + guard.cleanup().await; +} + +#[tokio::test] +async fn tls_skip_connect_relays_opaque_bytes_bidirectionally() { + let server = EchoServer::start().await.expect("start TCP echo server"); + let policy = write_policy(TEST_SERVER_HOST, server.port, " tls: skip") + .expect("write tls: skip policy"); + let policy_path = policy_path(&policy); + let script = format!( + r#" +import os +import socket +import urllib.parse + +HOST = {host:?} +PORT = {port} +PAYLOAD = bytes([0x00, 0xff, 0x13, 0x37, 0x80, 0x0a]) + b"not-http-or-tls" + bytes(range(64)) + +proxy_url = next( + os.environ[name] + for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") + if os.environ.get(name) +) +parsed = urllib.parse.urlparse(proxy_url) +with socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) as sock: + target = f"{{HOST}}:{{PORT}}" + sock.sendall(f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode()) + response = b"" + while b"\r\n\r\n" not in response: + response += sock.recv(4096) + if int(response.split(None, 2)[1]) != 200: + raise RuntimeError("CONNECT was denied") + sock.sendall(PAYLOAD) + echoed = b"" + while len(echoed) < len(PAYLOAD): + chunk = sock.recv(len(PAYLOAD) - len(echoed)) + if not chunk: + break + echoed += chunk + if echoed != PAYLOAD: + raise RuntimeError("opaque payload changed in transit") +print("RAW_RELAY_OK") +"#, + host = TEST_SERVER_HOST, + port = server.port, + ); + + let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script]) + .await + .expect("sandbox create"); + assert!( + guard.create_output.contains("RAW_RELAY_OK"), + "raw relay did not preserve the opaque payload:\n{}", + guard.create_output + ); +} + +#[tokio::test] +async fn http_credentials_are_rewritten_in_headers_and_bodies_for_both_adapters() { + let _provider_lock = PROVIDER_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + delete_provider(PROVIDER_NAME).await; + create_generic_provider(PROVIDER_NAME) + .await + .expect("create generic provider"); + + let result = async { + let server = CredentialProbeServer::start().await?; + let endpoint_options = r#" protocol: rest + enforcement: enforce + request_body_credential_rewrite: true + rules: + - allow: + method: POST + path: "/probe""#; + let policy = write_policy(TEST_SERVER_HOST, server.port, endpoint_options)?; + let policy_path = policy_path(&policy); + let script = format!( + r#" +import json +import os +import socket +import urllib.parse + +HOST = {host:?} +PORT = {port} +TOKEN = os.environ[{token_env:?}] + +def proxy_parts(): + proxy_url = next( + os.environ[name] + for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") + if os.environ.get(name) + ) + parsed = urllib.parse.urlparse(proxy_url) + return parsed.hostname, parsed.port or 80 + +def read_response(sock): + data = b"" + while b"\r\n\r\n" not in data: + chunk = sock.recv(4096) + if not chunk: + raise RuntimeError("incomplete response headers") + data += chunk + headers, body = data.split(b"\r\n\r\n", 1) + length = 0 + for line in headers.split(b"\r\n")[1:]: + if line.lower().startswith(b"content-length:"): + length = int(line.split(b":", 1)[1].strip()) + while len(body) < length: + chunk = sock.recv(4096) + if not chunk: + break + body += chunk + code = int(headers.split(None, 2)[1]) + if code != 200: + raise RuntimeError(f"request failed with HTTP {{code}}") + return json.loads(body[:length]) + +def request_bytes(target): + body = json.dumps({{"credential": TOKEN}}, separators=(",", ":")).encode() + return ( + f"POST {{target}} HTTP/1.1\r\n" + f"Host: {{HOST}}:{{PORT}}\r\n" + f"Authorization: Bearer {{TOKEN}}\r\n" + "Content-Type: application/json\r\n" + f"Content-Length: {{len(body)}}\r\n" + "Connection: close\r\n\r\n" + ).encode() + body + +proxy_host, proxy_port = proxy_parts() +target = f"{{HOST}}:{{PORT}}" +with socket.create_connection((proxy_host, proxy_port), timeout=10) as forward_sock: + forward_sock.sendall(request_bytes(f"http://{{target}}/probe")) + forward = read_response(forward_sock) + +with socket.create_connection((proxy_host, proxy_port), timeout=10) as connect_sock: + connect_sock.sendall(f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode()) + connect_response = b"" + while b"\r\n\r\n" not in connect_response: + connect_response += connect_sock.recv(4096) + if int(connect_response.split(None, 2)[1]) != 200: + raise RuntimeError("CONNECT was denied") + connect_sock.sendall(request_bytes("/probe")) + connect = read_response(connect_sock) + +print(json.dumps({{"connect": connect, "forward": forward}}, sort_keys=True)) +"#, + host = TEST_SERVER_HOST, + port = server.port, + token_env = TOKEN_ENV, + ); + + SandboxGuard::create(&[ + "--policy", + &policy_path, + "--provider", + PROVIDER_NAME, + "--", + "python3", + "-c", + &script, + ]) + .await + } + .await; + + delete_provider(PROVIDER_NAME).await; + + let guard = result.expect("sandbox create"); + let result = parse_json_line(&guard.create_output); + for adapter in ["connect", "forward"] { + assert_eq!( + result[adapter]["header_resolved"], true, + "{adapter} header placeholder was not resolved: {result}" + ); + assert_eq!( + result[adapter]["body_resolved"], true, + "{adapter} body placeholder was not resolved: {result}" + ); + assert_eq!( + result[adapter]["saw_placeholder"], false, + "{adapter} leaked an unresolved placeholder upstream: {result}" + ); + } + assert!( + !guard.create_output.contains(TEST_SECRET), + "sandbox output exposed the raw provider credential:\n{}", + guard.create_output + ); + assert!( + !guard.create_output.contains(PLACEHOLDER_PREFIX), + "sandbox output exposed an unresolved provider placeholder:\n{}", + guard.create_output + ); +} From 9673fed8dd15ce120ecfd01325a674cbb229035f Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Wed, 15 Jul 2026 12:06:36 -0700 Subject: [PATCH 03/30] refactor(network): make destination authorization explicit Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- .../openshell-supervisor-network/src/proxy.rs | 90 ++++++++-- .../src/proxy/destination.rs | 170 ++++++++++++------ .../src/proxy/egress.rs | 8 +- 3 files changed, 191 insertions(+), 77 deletions(-) diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index b02fda9a98..de6ca58144 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -36,7 +36,8 @@ use tokio::task::JoinHandle; use tracing::{debug, warn}; use self::destination::{ - DestinationDenial, DestinationDenialKind, DestinationRequest, validate_destination, + DestinationDenial, DestinationDenialKind, DestinationRequest, build_validation_plan, + validate_destination, }; use self::egress::{ EgressDecision, EgressIntent, EndpointDecision, IdentityUnavailableReason, L7ConfigSnapshot, @@ -1188,18 +1189,40 @@ async fn handle_tcp_connection( let sandbox_entrypoint_pid = entrypoint_pid.load(Ordering::Acquire); - hydrate_destination_constraints(&opa_engine, &mut decision); + match hydrate_destination_plan(&opa_engine, &mut decision, *trusted_host_gateway) { + Ok(()) => {} + Err(denial) => { + deny_connect_destination( + &mut client, + &denial, + peer_addr, + &host_lc, + port, + &binary_str, + &pid_str, + &ancestors_str, + &cmdline_str, + &decision, + &denial_tx, + &activity_tx, + ) + .await?; + return Ok(()); + } + } + let destination_plan = decision + .endpoint + .destination + .as_ref() + .expect("destination plan hydrated"); // Defense-in-depth: resolve DNS and reject connections to internal IPs. let dns_connect_start = std::time::Instant::now(); let connector = match validate_destination(DestinationRequest { host: &host, - normalized_host: &host_lc, port, sandbox_entrypoint_pid, - trusted_host_gateway: *trusted_host_gateway, - raw_allowed_ips: decision.endpoint.raw_allowed_ips.clone(), - exact_declared_endpoint_host: decision.endpoint.exact_declared_host, + plan: destination_plan, }) .await { @@ -2439,12 +2462,24 @@ fn hydrate_tls_mode(engine: &OpaEngine, decision: &mut EgressDecision) { decision.endpoint.tls_mode = query_tls_mode(engine, decision, &host, port); } -fn hydrate_destination_constraints(engine: &OpaEngine, decision: &mut EgressDecision) { +fn hydrate_destination_plan( + engine: &OpaEngine, + decision: &mut EgressDecision, + trusted_host_gateway: Option, +) -> std::result::Result<(), DestinationDenial> { let host = decision.intent.destination.host.clone(); let port = decision.intent.destination.port; - decision.endpoint.raw_allowed_ips = query_allowed_ips(engine, decision, &host, port); - decision.endpoint.exact_declared_host = - query_exact_declared_endpoint_host(engine, decision, &host, port); + let raw_allowed_ips = query_allowed_ips(engine, decision, &host, port); + let exact_declared_host = query_exact_declared_endpoint_host(engine, decision, &host, port); + let plan = build_validation_plan( + &host, + &host.to_ascii_lowercase(), + trusted_host_gateway, + &raw_allowed_ips, + exact_declared_host, + )?; + decision.endpoint.destination = Some(plan); + Ok(()) } fn query_l7_route_snapshot( @@ -4334,16 +4369,41 @@ async fn handle_forward_proxy( // - Otherwise: reject internal IPs, allow public IPs through. // When the policy host is already a literal IP address, treat it as // implicitly allowed — the user explicitly declared the destination. - hydrate_destination_constraints(&opa_engine, &mut decision); + match hydrate_destination_plan(&opa_engine, &mut decision, *trusted_host_gateway) { + Ok(()) => {} + Err(denial) => { + deny_forward_destination( + client, + &denial, + peer_addr, + method, + &host_lc, + port, + &path, + &binary_str, + &pid_str, + &ancestors_str, + &cmdline_str, + policy_str, + &decision, + denial_tx, + activity_tx, + ) + .await?; + return Ok(()); + } + } + let destination_plan = decision + .endpoint + .destination + .as_ref() + .expect("destination plan hydrated"); let connector = match validate_destination(DestinationRequest { host: &host, - normalized_host: &host_lc, port, sandbox_entrypoint_pid, - trusted_host_gateway: *trusted_host_gateway, - raw_allowed_ips: decision.endpoint.raw_allowed_ips.clone(), - exact_declared_endpoint_host: decision.endpoint.exact_declared_host, + plan: destination_plan, }) .await { diff --git a/crates/openshell-supervisor-network/src/proxy/destination.rs b/crates/openshell-supervisor-network/src/proxy/destination.rs index b9fc42cf75..2653afd78f 100644 --- a/crates/openshell-supervisor-network/src/proxy/destination.rs +++ b/crates/openshell-supervisor-network/src/proxy/destination.rs @@ -8,18 +8,32 @@ use super::{ resolve_and_check_allowed_ips, resolve_and_check_declared_endpoint, resolve_and_check_trusted_gateway, resolve_and_reject_internal, }; +use ipnet::IpNet; use std::net::{IpAddr, SocketAddr}; use tokio::net::TcpStream; +/// Address-validation mode selected from the current endpoint configuration. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) enum AddressAuthorization { + DefaultPublicOnly, + ExplicitAllowedIps(Vec), + ExactDeclaredHost, + ImplicitIpLiteral(IpAddr), + TrustedGatewayAlias { expected_ip: IpAddr }, +} + +/// Fully materialized input to shared destination validation. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct DestinationValidationPlan { + pub(super) address_authorization: AddressAuthorization, +} + /// Inputs needed to apply the current SSRF and endpoint destination policy. pub(super) struct DestinationRequest<'a> { pub(super) host: &'a str, - pub(super) normalized_host: &'a str, pub(super) port: u16, pub(super) sandbox_entrypoint_pid: u32, - pub(super) trusted_host_gateway: Option, - pub(super) raw_allowed_ips: Vec, - pub(super) exact_declared_endpoint_host: bool, + pub(super) plan: &'a DestinationValidationPlan, } /// Destination-validation branch that rejected an egress request. @@ -47,6 +61,38 @@ impl DestinationDenial { } } +/// Select one current destination-validation mode without changing precedence. +pub(super) fn build_validation_plan( + host: &str, + normalized_host: &str, + trusted_host_gateway: Option, + raw_allowed_ips: &[String], + exact_declared_endpoint_host: bool, +) -> Result { + let address_authorization = if is_host_gateway_alias(normalized_host) + && let Some(expected_ip) = trusted_host_gateway + { + AddressAuthorization::TrustedGatewayAlias { expected_ip } + } else if !raw_allowed_ips.is_empty() { + AddressAuthorization::ExplicitAllowedIps(parse_allowed_ips(raw_allowed_ips).map_err( + |reason| DestinationDenial::new(DestinationDenialKind::InvalidAllowedIps, reason), + )?) + } else if let Some(ip) = implicit_allowed_ips_for_ip_host(host) + .first() + .and_then(|raw| raw.parse::().ok()) + { + AddressAuthorization::ImplicitIpLiteral(ip) + } else if exact_declared_endpoint_host { + AddressAuthorization::ExactDeclaredHost + } else { + AddressAuthorization::DefaultPublicOnly + }; + + Ok(DestinationValidationPlan { + address_authorization, + }) +} + /// Validated, but not yet opened, upstream destination. /// /// The explicit proxy adapter controls when `connect` is called so CONNECT and @@ -87,46 +133,48 @@ pub(super) async fn validate_destination( ) -> Result { let DestinationRequest { host, - normalized_host, port, sandbox_entrypoint_pid, - trusted_host_gateway, - mut raw_allowed_ips, - exact_declared_endpoint_host, + plan, } = request; - if raw_allowed_ips.is_empty() { - raw_allowed_ips = implicit_allowed_ips_for_ip_host(host); - } - - #[allow(clippy::if_not_else)] - let addrs = if is_host_gateway_alias(normalized_host) - && let Some(gateway) = trusted_host_gateway - { - resolve_and_check_trusted_gateway(host, port, gateway, sandbox_entrypoint_pid) - .await - .map_err(|reason| { - DestinationDenial::new(DestinationDenialKind::TrustedGateway, reason) - })? - } else if !raw_allowed_ips.is_empty() { - let networks = parse_allowed_ips(&raw_allowed_ips).map_err(|reason| { - DestinationDenial::new(DestinationDenialKind::InvalidAllowedIps, reason) - })?; - resolve_and_check_allowed_ips(host, port, &networks, sandbox_entrypoint_pid) - .await - .map_err(|reason| DestinationDenial::new(DestinationDenialKind::AllowedIps, reason))? - } else if exact_declared_endpoint_host { - resolve_and_check_declared_endpoint(host, port, sandbox_entrypoint_pid) - .await - .map_err(|reason| { - DestinationDenial::new(DestinationDenialKind::DeclaredEndpoint, reason) - })? - } else { - resolve_and_reject_internal(host, port, sandbox_entrypoint_pid) - .await - .map_err(|reason| { - DestinationDenial::new(DestinationDenialKind::InternalAddress, reason) - })? + let addrs = match &plan.address_authorization { + AddressAuthorization::TrustedGatewayAlias { expected_ip } => { + resolve_and_check_trusted_gateway(host, port, *expected_ip, sandbox_entrypoint_pid) + .await + .map_err(|reason| { + DestinationDenial::new(DestinationDenialKind::TrustedGateway, reason) + })? + } + AddressAuthorization::ExplicitAllowedIps(networks) => { + resolve_and_check_allowed_ips(host, port, networks, sandbox_entrypoint_pid) + .await + .map_err(|reason| { + DestinationDenial::new(DestinationDenialKind::AllowedIps, reason) + })? + } + AddressAuthorization::ImplicitIpLiteral(ip) => { + let network = IpNet::from(*ip); + resolve_and_check_allowed_ips(host, port, &[network], sandbox_entrypoint_pid) + .await + .map_err(|reason| { + DestinationDenial::new(DestinationDenialKind::AllowedIps, reason) + })? + } + AddressAuthorization::ExactDeclaredHost => { + resolve_and_check_declared_endpoint(host, port, sandbox_entrypoint_pid) + .await + .map_err(|reason| { + DestinationDenial::new(DestinationDenialKind::DeclaredEndpoint, reason) + })? + } + AddressAuthorization::DefaultPublicOnly => { + resolve_and_reject_internal(host, port, sandbox_entrypoint_pid) + .await + .map_err(|reason| { + DestinationDenial::new(DestinationDenialKind::InternalAddress, reason) + })? + } }; Ok(UpstreamConnector::new(host, port, addrs)) @@ -137,21 +185,21 @@ mod tests { use super::*; use std::net::{IpAddr, Ipv4Addr}; - fn request(host: &str) -> DestinationRequest<'_> { + fn request<'a>(host: &'a str, plan: &'a DestinationValidationPlan) -> DestinationRequest<'a> { DestinationRequest { host, - normalized_host: host, port: 80, sandbox_entrypoint_pid: 0, - trusted_host_gateway: None, - raw_allowed_ips: vec![], - exact_declared_endpoint_host: false, + plan, } } #[tokio::test] async fn default_mode_classifies_loopback_as_internal_address() { - let denial = validate_destination(request("127.0.0.1")) + let plan = DestinationValidationPlan { + address_authorization: AddressAuthorization::DefaultPublicOnly, + }; + let denial = validate_destination(request("127.0.0.1", &plan)) .await .err() .expect("loopback must be denied"); @@ -161,21 +209,24 @@ mod tests { #[tokio::test] async fn invalid_allowed_ips_has_a_distinct_denial_kind() { - let mut request = request("api.example.test"); - request.raw_allowed_ips = vec!["not-an-ip".to_string()]; - let denial = validate_destination(request) - .await - .err() - .expect("invalid allowed_ips must be denied"); + let denial = build_validation_plan( + "api.example.test", + "api.example.test", + None, + &["not-an-ip".to_string()], + false, + ) + .expect_err("invalid allowed_ips must be denied"); assert_eq!(denial.kind, DestinationDenialKind::InvalidAllowedIps); } #[tokio::test] async fn declared_endpoint_preserves_its_denial_classification() { - let mut request = request("127.0.0.1"); - request.exact_declared_endpoint_host = true; - let denial = validate_destination(request) + let plan = DestinationValidationPlan { + address_authorization: AddressAuthorization::ExactDeclaredHost, + }; + let denial = validate_destination(request("127.0.0.1", &plan)) .await .err() .expect("declared loopback must remain denied"); @@ -185,9 +236,12 @@ mod tests { #[tokio::test] async fn trusted_gateway_preserves_its_denial_classification() { - let mut request = request("host.openshell.internal"); - request.trusted_host_gateway = Some(IpAddr::V4(Ipv4Addr::LOCALHOST)); - let denial = validate_destination(request) + let plan = DestinationValidationPlan { + address_authorization: AddressAuthorization::TrustedGatewayAlias { + expected_ip: IpAddr::V4(Ipv4Addr::LOCALHOST), + }, + }; + let denial = validate_destination(request("host.openshell.internal", &plan)) .await .err() .expect("loopback cannot be a trusted gateway"); diff --git a/crates/openshell-supervisor-network/src/proxy/egress.rs b/crates/openshell-supervisor-network/src/proxy/egress.rs index 3d57dbe4ec..712c04c817 100644 --- a/crates/openshell-supervisor-network/src/proxy/egress.rs +++ b/crates/openshell-supervisor-network/src/proxy/egress.rs @@ -9,6 +9,7 @@ //! independent of CONNECT and forward HTTP prevents policy behavior from //! drifting as more adapters are added. +use super::destination::DestinationValidationPlan; use crate::opa::NetworkAction; use std::path::PathBuf; @@ -31,8 +32,8 @@ pub(super) struct L7RouteSnapshot { pub(super) struct EndpointDecision { pub(super) tls_mode: crate::l7::TlsMode, pub(super) l7_route: Option, - pub(super) raw_allowed_ips: Vec, - pub(super) exact_declared_host: bool, + /// Destination authorization selected at the legacy hydration point. + pub(super) destination: Option, } impl Default for EndpointDecision { @@ -40,8 +41,7 @@ impl Default for EndpointDecision { Self { tls_mode: crate::l7::TlsMode::Auto, l7_route: None, - raw_allowed_ips: vec![], - exact_declared_host: false, + destination: None, } } } From ff9a6f8740ee6159a2c6c6c604c7647635c17039 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Wed, 15 Jul 2026 12:12:40 -0700 Subject: [PATCH 04/30] refactor(network): pin proxy relay policy context Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- architecture/sandbox.md | 14 +- .../openshell-supervisor-network/src/proxy.rs | 95 +++++----- .../src/proxy/egress.rs | 5 +- .../src/proxy/relay.rs | 168 ++++++++++++------ 4 files changed, 173 insertions(+), 109 deletions(-) diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 95be47569a..a39f699a57 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -60,12 +60,16 @@ socket inode. CONNECT and absolute-form forward HTTP are explicit-proxy adapters over the same egress pipeline. Each adapter normalizes its request into an egress intent, and -the shared authorization result carries the process evidence and endpoint state -used by destination validation and relay selection. Destination validation +the shared authorization result carries the process evidence used by destination +validation and relay selection. During the compatibility migration, endpoint +state is hydrated at the adapters' existing policy query points; it is not yet +one atomic, generation-consistent authorization result. Destination validation returns an unopened connector so adapters retain their existing response and -upstream-dial timing. CONNECT uses shared TLS-terminated HTTP, plaintext HTTP, -and raw byte relay primitives. Forward HTTP retains its guarded single-request -relay while sharing authorization, request context, and destination boundaries. +upstream-dial timing. CONNECT prepares a generation-pinned relay context before +entering shared TLS-terminated or plaintext HTTP relays; non-HTTP traffic uses +the shared raw byte relay after the existing adapter gates. Forward HTTP retains +its guarded single-request relay while sharing authorization, request context, +policy-pinning, and destination boundaries. Adapter-specific response and OCSF event shapes remain at the protocol boundary. For inspected HTTP traffic, the proxy can enforce REST method/path rules, diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index de6ca58144..6286e481e4 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -1086,7 +1086,7 @@ async fn handle_tcp_connection( let pid_clone = entrypoint_pid.clone(); let intent = EgressIntent::connect(host_lc.clone(), port); let mut decision = tokio::task::spawn_blocking(move || { - evaluate_opa_tcp(connection, &opa_clone, &cache_clone, &pid_clone, intent) + authorize_egress_intent(connection, &opa_clone, &cache_clone, &pid_clone, intent) }) .await .map_err(|e| miette::miette!("identity resolution task panicked: {e}"))?; @@ -1403,16 +1403,13 @@ async fn handle_tcp_connection( let mut tls_upstream = crate::l7::tls::tls_connect_upstream(upstream, &host_lc, tls.upstream_config()) .await?; + let Some(relay_context) = + relay::prepare_http_relay(l7_route, &opa_engine, &decision, &ctx) + else { + return Ok(()); + }; - relay::relay_http_stream( - l7_route, - &opa_engine, - &decision, - &mut tls_client, - &mut tls_upstream, - &ctx, - ) - .await + relay::relay_http_stream(&mut tls_client, &mut tls_upstream, relay_context).await }; if let Err(e) = tls_result.await { if is_benign_relay_error(&e) { @@ -1480,16 +1477,11 @@ async fn handle_tcp_connection( } else if tunnel_protocol == TunnelProtocol::Http1 { // Plaintext HTTP detected. let is_l7_relay = l7_route.is_some_and(|route| !route.configs.is_empty()); - if let Err(e) = relay::relay_http_stream( - l7_route, - &opa_engine, - &decision, - &mut client, - &mut upstream, - &ctx, - ) - .await - { + let Some(relay_context) = relay::prepare_http_relay(l7_route, &opa_engine, &decision, &ctx) + else { + return Ok(()); + }; + if let Err(e) = relay::relay_http_stream(&mut client, &mut upstream, relay_context).await { if is_benign_relay_error(&e) { if is_l7_relay { debug!(host = %host_lc, port = port, error = %e, "L7 connection closed"); @@ -1581,7 +1573,7 @@ async fn handle_tcp_connection( /// Resolved process identity for a TCP peer: binary path, PID, ancestor chain, /// cmdline paths, and the TOFU-verified binary hash. /// -/// Produced by [`resolve_process_identity`]; consumed by [`evaluate_opa_tcp`] +/// Produced by [`resolve_process_identity`]; consumed by [`authorize_egress_intent`] /// and by the identity-chain regression tests. #[cfg(target_os = "linux")] struct ResolvedIdentity { @@ -1718,7 +1710,7 @@ fn collect_ancestor_identities(start_pid: u32, stop_pid: u32) -> Vec<(u32, PathB /// walks each ancestor chain verifying every ancestor, and collects /// cmdline-derived absolute paths for script detection. /// -/// This is the identity-resolution block of [`evaluate_opa_tcp`] extracted +/// This is the identity-resolution block of [`authorize_egress_intent`] extracted /// into a standalone helper so it can be exercised by Linux-only regression /// tests without a full OPA engine. The key hot-swap invariant under test is /// that display paths are stripped for policy/logging, while integrity hashing @@ -1794,7 +1786,7 @@ fn resolve_process_identity( /// Evaluate OPA policy for a TCP connection with identity binding via /proc/net/tcp. #[cfg(target_os = "linux")] -fn evaluate_opa_tcp( +fn authorize_egress_intent( connection: crate::procfs::WorkloadProxyTcpConnection, engine: &OpaEngine, identity_cache: &BinaryIdentityCache, @@ -1814,7 +1806,7 @@ fn evaluate_opa_tcp( EgressDecision { intent: intent.clone(), action: NetworkAction::Deny { reason }, - generation: engine.current_generation(), + l4_policy_generation: engine.current_generation(), identity, endpoint: EndpointDecision::default(), binary, @@ -1827,7 +1819,7 @@ fn evaluate_opa_tcp( if !crate::opa::network_binary_identity_required() { let result = evaluate_endpoint_only_opa(engine, intent); debug!( - "evaluate_opa_tcp endpoint-only: host={} port={} transport={:?} action={:?}", + "authorize_egress_intent endpoint-only: host={} port={} transport={:?} action={:?}", result.intent.destination.host, result.intent.destination.port, result.intent.transport, @@ -1884,7 +1876,7 @@ fn evaluate_opa_tcp( Ok((action, generation)) => EgressDecision { intent: intent.clone(), action, - generation, + l4_policy_generation: generation, identity: ProcessIdentityEvidence::Available, endpoint: EndpointDecision::default(), binary: Some(bin_path), @@ -1902,7 +1894,7 @@ fn evaluate_opa_tcp( ), }; debug!( - "evaluate_opa_tcp TOTAL: {}ms host={} port={} transport={:?}", + "authorize_egress_intent TOTAL: {}ms host={} port={} transport={:?}", total_start.elapsed().as_millis(), intent.destination.host, intent.destination.port, @@ -1939,7 +1931,7 @@ fn evaluate_endpoint_only_opa(engine: &OpaEngine, intent: EgressIntent) -> Egres Ok((action, generation)) => EgressDecision { intent, action, - generation, + l4_policy_generation: generation, identity: ProcessIdentityEvidence::Unavailable( IdentityUnavailableReason::EndpointOnlyMode, ), @@ -1954,7 +1946,7 @@ fn evaluate_endpoint_only_opa(engine: &OpaEngine, intent: EgressIntent) -> Egres action: NetworkAction::Deny { reason: format!("policy evaluation error: {e}"), }, - generation: engine.current_generation(), + l4_policy_generation: engine.current_generation(), identity: ProcessIdentityEvidence::Unavailable( IdentityUnavailableReason::EndpointOnlyMode, ), @@ -1969,7 +1961,7 @@ fn evaluate_endpoint_only_opa(engine: &OpaEngine, intent: EgressIntent) -> Egres /// Non-Linux stub: OPA identity binding requires /proc. #[cfg(not(target_os = "linux"))] -fn evaluate_opa_tcp( +fn authorize_egress_intent( _connection: crate::procfs::WorkloadProxyTcpConnection, engine: &OpaEngine, _identity_cache: &BinaryIdentityCache, @@ -1985,7 +1977,7 @@ fn evaluate_opa_tcp( action: NetworkAction::Deny { reason: "identity binding unavailable on this platform".into(), }, - generation: engine.current_generation(), + l4_policy_generation: engine.current_generation(), identity: ProcessIdentityEvidence::Unavailable( IdentityUnavailableReason::UnsupportedPlatform, ), @@ -2446,7 +2438,7 @@ fn emit_l7_tunnel_close_after_policy_change(host: &str, port: u16, error: miette ocsf_emit!(event); } -/// Query L7 endpoint config from the OPA engine for a matched CONNECT decision. +/// Query L7 endpoint config from the OPA engine for an allowed egress decision. /// /// Returns `Some(L7EndpointConfig)` if the matched endpoint has L7 config (protocol field), /// `None` for L4-only endpoints. @@ -2522,7 +2514,7 @@ fn query_l7_route_snapshot( ); Some(L7RouteSnapshot { configs, - generation, + l7_policy_generation: generation, }) } Err(e) => { @@ -3773,7 +3765,7 @@ async fn handle_forward_proxy( let pid_clone = entrypoint_pid.clone(); let intent = EgressIntent::forward_http(host_lc.clone(), port); let mut decision = tokio::task::spawn_blocking(move || { - evaluate_opa_tcp(connection, &opa_clone, &cache_clone, &pid_clone, intent) + authorize_egress_intent(connection, &opa_clone, &cache_clone, &pid_clone, intent) }) .await .map_err(|e| miette::miette!("identity resolution task panicked: {e}"))?; @@ -3869,19 +3861,22 @@ async fn handle_forward_proxy( binary = %binary_str, binary_pid = %pid_str, matched_policy = %policy_str, - decision_generation = decision.generation, + l4_policy_generation = decision.l4_policy_generation, current_generation = opa_engine.current_generation(), action = ?decision.action, "Forward proxy L4 policy decision" ); let sandbox_entrypoint_pid = entrypoint_pid.load(Ordering::Acquire); - let forward_generation_guard = match opa_engine.generation_guard(decision.generation) { + let forward_generation_guard = match relay::pin_policy_generation( + &opa_engine, + decision.l4_policy_generation, + ) { Ok(guard) => guard, Err(e) => { warn!( host = %host_lc, port, - decision_generation = decision.generation, + l4_policy_generation = decision.l4_policy_generation, current_generation = opa_engine.current_generation(), error = %e, "Forward proxy rejected request because policy generation changed after L4 decision" @@ -3934,13 +3929,13 @@ async fn handle_forward_proxy( .as_ref() .filter(|route| !route.configs.is_empty()) { - if route.generation != forward_generation_guard.captured_generation() { + if route.l7_policy_generation != forward_generation_guard.captured_generation() { warn!( host = %host_lc, port, - decision_generation = decision.generation, - guard_generation = forward_generation_guard.captured_generation(), - route_generation = route.generation, + l4_policy_generation = decision.l4_policy_generation, + l4_guard_generation = forward_generation_guard.captured_generation(), + l7_policy_generation = route.l7_policy_generation, current_generation = opa_engine.current_generation(), "Forward proxy rejected request because L7 route lookup used a different policy generation" ); @@ -3950,7 +3945,7 @@ async fn handle_forward_proxy( miette::miette!( "policy changed before forward L7 evaluation [expected_generation:{} current_generation:{}]", forward_generation_guard.captured_generation(), - route.generation, + route.l7_policy_generation, ), ); emit_activity_simple(activity_tx, true, "policy_stale"); @@ -3966,13 +3961,13 @@ async fn handle_forward_proxy( .await?; return Ok(()); } - let tunnel_engine = match opa_engine.clone_engine_for_tunnel(route.generation) { + let tunnel_engine = match relay::pin_l7_evaluator(&opa_engine, route.l7_policy_generation) { Ok(engine) => engine, Err(e) => { warn!( host = %host_lc, port, - route_generation = route.generation, + l7_policy_generation = route.l7_policy_generation, current_generation = opa_engine.current_generation(), error = %e, "Forward proxy rejected request because L7 tunnel engine could not be cloned" @@ -5334,11 +5329,11 @@ network_policies: configs: vec![L7ConfigSnapshot { config: websocket_l7_config(crate::l7::L7Protocol::Rest, false), }], - generation: 1, + l7_policy_generation: 1, }; let l4_route = L7RouteSnapshot { configs: Vec::new(), - generation: 1, + l7_policy_generation: 1, }; emit_connect_activity_if_l4_only(&activity_tx, Some(&l7_route)); @@ -5749,7 +5744,7 @@ network_policies: action: NetworkAction::Allow { matched_policy: Some(policy_name.to_string()), }, - generation: engine.current_generation(), + l4_policy_generation: engine.current_generation(), identity: ProcessIdentityEvidence::Available, endpoint: EndpointDecision::default(), binary: Some(PathBuf::from("/usr/bin/node")), @@ -5764,7 +5759,7 @@ network_policies: .config .clone(); let tunnel_engine = engine - .clone_engine_for_tunnel(route.generation) + .clone_engine_for_tunnel(route.l7_policy_generation) .expect("tunnel engine"); let ctx = crate::l7::relay::L7EvalContext { host: host.to_string(), @@ -9263,7 +9258,7 @@ network_policies: /// itself), binds to `current_exe()`, and never falls through to the /// whole-`/proc` scan — the environment-sensitive path that made a forked /// child flaky under a busy CI `/proc`. Callers gate on Linux; - /// `evaluate_opa_tcp` denies unconditionally without `/proc`. + /// `authorize_egress_intent` denies unconditionally without `/proc`. async fn drive_connect_through_handler( endpoint_yaml: &str, connect_target: &str, @@ -9576,7 +9571,7 @@ network_policies: let decision = EgressDecision { intent: EgressIntent::connect("203.0.113.10".to_string(), 443), action, - generation, + l4_policy_generation: generation, identity: ProcessIdentityEvidence::Available, endpoint: EndpointDecision::default(), binary: Some(input.binary_path), diff --git a/crates/openshell-supervisor-network/src/proxy/egress.rs b/crates/openshell-supervisor-network/src/proxy/egress.rs index 712c04c817..bf1b05131b 100644 --- a/crates/openshell-supervisor-network/src/proxy/egress.rs +++ b/crates/openshell-supervisor-network/src/proxy/egress.rs @@ -21,7 +21,8 @@ pub(super) struct L7ConfigSnapshot { #[derive(Debug, Clone)] pub(super) struct L7RouteSnapshot { pub(super) configs: Vec, - pub(super) generation: u64, + /// Policy generation used to materialize this L7 route. + pub(super) l7_policy_generation: u64, } /// Endpoint metadata materialized for an allowed egress decision. @@ -111,7 +112,7 @@ pub(super) struct EgressDecision { pub(super) intent: EgressIntent, pub(super) action: NetworkAction, /// Policy generation used for the L4 network decision. - pub(super) generation: u64, + pub(super) l4_policy_generation: u64, /// Whether process identity evidence was available to policy evaluation. pub(super) identity: ProcessIdentityEvidence, /// Endpoint behavior hydrated for destination validation and relays. diff --git a/crates/openshell-supervisor-network/src/proxy/relay.rs b/crates/openshell-supervisor-network/src/proxy/relay.rs index 68de77dcfd..6917cf3f35 100644 --- a/crates/openshell-supervisor-network/src/proxy/relay.rs +++ b/crates/openshell-supervisor-network/src/proxy/relay.rs @@ -5,7 +5,7 @@ use super::{EgressDecision, L7RouteSnapshot, emit_l7_tunnel_close_after_policy_change}; use crate::l7::relay::L7EvalContext; -use crate::opa::{NetworkAction, OpaEngine}; +use crate::opa::{NetworkAction, OpaEngine, PolicyGenerationGuard, TunnelPolicyEngine}; use miette::{IntoDiagnostic, Result}; use openshell_core::activity::ActivitySender; use openshell_core::proto::ProviderProfileCredential; @@ -16,6 +16,27 @@ use tokio::io::{AsyncRead, AsyncWrite}; type DynamicCredentials = Arc>>; +enum PreparedHttpPolicy { + Inspect { + configs: Vec, + evaluator: Box, + }, + Passthrough { + generation_guard: PolicyGenerationGuard, + }, +} + +/// Everything an HTTP relay needs after authorization is complete. +/// +/// The relay deliberately owns a generation-pinned policy primitive instead +/// of retaining access to the mutable OPA engine. Policy reloads therefore +/// fail closed through the guard or tunnel evaluator already attached here. +pub(super) struct RelayContext<'a> { + request: &'a L7EvalContext, + policy: PreparedHttpPolicy, + middleware_engine: &'a OpaEngine, +} + /// Build the request-processing context shared by CONNECT and forward HTTP. pub(super) fn http_context( decision: &EgressDecision, @@ -58,81 +79,124 @@ pub(super) fn http_context( } } -/// Relay an HTTP/1 stream using the endpoint's current L7 configuration. +/// Pin a generation for a relay or the forward HTTP single-request path. +pub(super) fn pin_policy_generation( + opa_engine: &OpaEngine, + expected_generation: u64, +) -> Result { + opa_engine.generation_guard(expected_generation) +} + +/// Clone an L7 evaluator for a relay or the forward HTTP single-request path. +pub(super) fn pin_l7_evaluator( + opa_engine: &OpaEngine, + expected_generation: u64, +) -> Result { + opa_engine.clone_engine_for_tunnel(expected_generation) +} + +/// Prepare a generation-pinned HTTP relay at the adapter boundary. /// -/// CONNECT plaintext and TLS-terminated streams both enter through this -/// function. Forward HTTP will provide a buffered first request to the same -/// boundary in the next migration step. -pub(super) async fn relay_http_stream( +/// A stale generation preserves the established CONNECT behavior: emit the +/// policy-change close event and let the adapter close the live tunnel without +/// attempting to write an HTTP response into it. +pub(super) fn prepare_http_relay<'a>( route: Option<&L7RouteSnapshot>, - opa_engine: &Arc, + opa_engine: &OpaEngine, decision: &EgressDecision, - client: &mut C, - upstream: &mut U, - context: &L7EvalContext, -) -> Result<()> -where - C: AsyncRead + AsyncWrite + Unpin + Send, - U: AsyncRead + AsyncWrite + Unpin + Send, -{ - if let Some(route) = route.filter(|route| !route.configs.is_empty()) { - let tunnel_engine = match opa_engine.clone_engine_for_tunnel(route.generation) { - Ok(engine) => engine, + request: &'a L7EvalContext, +) -> Option> { + let policy = if let Some(route) = route.filter(|route| !route.configs.is_empty()) { + let evaluator = match pin_l7_evaluator(opa_engine, route.l7_policy_generation) { + Ok(evaluator) => evaluator, + Err(error) => { + emit_l7_tunnel_close_after_policy_change( + &decision.intent.destination.host, + decision.intent.destination.port, + error, + ); + return None; + } + }; + let configs = route + .configs + .iter() + .map(|snapshot| snapshot.config.clone()) + .collect(); + PreparedHttpPolicy::Inspect { + configs, + evaluator: Box::new(evaluator), + } + } else { + let expected_generation = route.map_or(decision.l4_policy_generation, |route| { + route.l7_policy_generation + }); + let generation_guard = match pin_policy_generation(opa_engine, expected_generation) { + Ok(guard) => guard, Err(error) => { emit_l7_tunnel_close_after_policy_change( &decision.intent.destination.host, decision.intent.destination.port, error, ); - return Ok(()); + return None; } }; + PreparedHttpPolicy::Passthrough { generation_guard } + }; + + Some(RelayContext { + request, + policy, + middleware_engine: opa_engine, + }) +} - if route.configs.len() == 1 { +/// Relay an HTTP/1 stream using an already-authorized, generation-pinned context. +/// +/// CONNECT plaintext and TLS-terminated streams both enter through this +/// function. Forward HTTP will provide a buffered first request to the same +/// boundary in the next migration step. +pub(super) async fn relay_http_stream( + client: &mut C, + upstream: &mut U, + context: RelayContext<'_>, +) -> Result<()> +where + C: AsyncRead + AsyncWrite + Unpin + Send, + U: AsyncRead + AsyncWrite + Unpin + Send, +{ + match context.policy { + PreparedHttpPolicy::Inspect { configs, evaluator } if configs.len() == 1 => { crate::l7::relay::relay_with_inspection( - &route.configs[0].config, - tunnel_engine, + &configs[0], + *evaluator, client, upstream, - context, + context.request, ) .await - } else { - let configs = route - .configs - .iter() - .map(|snapshot| snapshot.config.clone()) - .collect::>(); + } + PreparedHttpPolicy::Inspect { configs, evaluator } => { crate::l7::relay::relay_with_route_selection( &configs, - tunnel_engine, + *evaluator, client, upstream, - context, + context.request, + ) + .await + } + PreparedHttpPolicy::Passthrough { generation_guard } => { + crate::l7::relay::relay_passthrough_with_credentials( + client, + upstream, + context.request, + &generation_guard, + Some(context.middleware_engine), ) .await } - } else { - let generation = route.map_or(decision.generation, |route| route.generation); - let generation_guard = match opa_engine.generation_guard(generation) { - Ok(guard) => guard, - Err(error) => { - emit_l7_tunnel_close_after_policy_change( - &decision.intent.destination.host, - decision.intent.destination.port, - error, - ); - return Ok(()); - } - }; - crate::l7::relay::relay_passthrough_with_credentials( - client, - upstream, - context, - &generation_guard, - Some(opa_engine), - ) - .await } } From e44072c2f8a7f36a2748104e6e2693e34ba89fe1 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Wed, 15 Jul 2026 12:15:58 -0700 Subject: [PATCH 05/30] test(network): lock relay generation contracts Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- .../src/proxy/relay.rs | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/crates/openshell-supervisor-network/src/proxy/relay.rs b/crates/openshell-supervisor-network/src/proxy/relay.rs index 6917cf3f35..b025b5ceb4 100644 --- a/crates/openshell-supervisor-network/src/proxy/relay.rs +++ b/crates/openshell-supervisor-network/src/proxy/relay.rs @@ -211,3 +211,96 @@ where .into_diagnostic()?; Ok(()) } + +#[cfg(test)] +mod tests { + use super::super::{EgressIntent, EndpointDecision, ProcessIdentityEvidence}; + use super::*; + + const POLICY_REGO: &str = include_str!("../../data/sandbox-policy.rego"); + const EMPTY_POLICY_DATA: &str = "network_policies: {}\n"; + + fn decision(l4_policy_generation: u64) -> EgressDecision { + EgressDecision { + intent: EgressIntent::connect("example.com".to_string(), 80), + action: NetworkAction::Allow { + matched_policy: Some("test".to_string()), + }, + l4_policy_generation, + identity: ProcessIdentityEvidence::Available, + endpoint: EndpointDecision::default(), + binary: None, + binary_pid: None, + ancestors: vec![], + cmdline_paths: vec![], + } + } + + fn request_context() -> L7EvalContext { + L7EvalContext { + host: "example.com".to_string(), + port: 80, + policy_name: "test".to_string(), + binary_path: String::new(), + ancestors: vec![], + cmdline_paths: vec![], + secret_resolver: None, + activity_tx: None, + dynamic_credentials: None, + token_grant_resolver: None, + } + } + + #[test] + fn relay_without_route_pins_l4_decision_generation() { + let engine = OpaEngine::from_strings(POLICY_REGO, EMPTY_POLICY_DATA).unwrap(); + let decision = decision(engine.current_generation()); + let request = request_context(); + + let context = prepare_http_relay(None, &engine, &decision, &request) + .expect("current L4 generation should prepare a relay"); + let PreparedHttpPolicy::Passthrough { generation_guard } = context.policy else { + panic!("route-less relay should use a generation guard"); + }; + + assert_eq!( + generation_guard.captured_generation(), + decision.l4_policy_generation + ); + } + + #[test] + fn empty_hydrated_route_pins_l7_lookup_generation() { + let engine = OpaEngine::from_strings(POLICY_REGO, EMPTY_POLICY_DATA).unwrap(); + let decision = decision(u64::MAX); + let route = L7RouteSnapshot { + configs: vec![], + l7_policy_generation: engine.current_generation(), + }; + let request = request_context(); + + let context = prepare_http_relay(Some(&route), &engine, &decision, &request) + .expect("the hydrated route generation should take precedence over stale L4 data"); + let PreparedHttpPolicy::Passthrough { generation_guard } = context.policy else { + panic!("empty L7 route should use a generation guard"); + }; + + assert_eq!( + generation_guard.captured_generation(), + route.l7_policy_generation + ); + } + + #[test] + fn stale_generation_fails_before_relay_context_is_created() { + let engine = OpaEngine::from_strings(POLICY_REGO, EMPTY_POLICY_DATA).unwrap(); + let decision = decision(engine.current_generation()); + let request = request_context(); + engine.reload(POLICY_REGO, EMPTY_POLICY_DATA).unwrap(); + + assert!( + prepare_http_relay(None, &engine, &decision, &request).is_none(), + "policy reload must prevent a stale relay from starting" + ); + } +} From 4185b114728ac85a31bf741b72df0f6db65d90fe Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Thu, 16 Jul 2026 15:06:52 -0700 Subject: [PATCH 06/30] test(network): establish phase zero compatibility baseline Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- .../openshell-sandbox/src/metadata_server.rs | 90 +++ .../openshell-supervisor-network/src/lib.rs | 60 ++ .../openshell-supervisor-network/src/opa.rs | 66 +- .../openshell-supervisor-network/src/proxy.rs | 167 ++++- .../src/proxy/destination.rs | 52 ++ .../src/proxy/tests/phase0.rs | 652 ++++++++++++++++++ .../src/bypass_monitor/mod.rs | 156 +++-- e2e/rust/tests/proxy_egress_pipeline.rs | 388 ++++++++++- e2e/rust/tests/websocket_conformance.rs | 69 +- 9 files changed, 1585 insertions(+), 115 deletions(-) create mode 100644 crates/openshell-supervisor-network/src/proxy/tests/phase0.rs diff --git a/crates/openshell-sandbox/src/metadata_server.rs b/crates/openshell-sandbox/src/metadata_server.rs index aff1b5418a..c618d8e771 100644 --- a/crates/openshell-sandbox/src/metadata_server.rs +++ b/crates/openshell-sandbox/src/metadata_server.rs @@ -135,3 +135,93 @@ async fn handle_connection( Ok(()) }) } + +#[cfg(test)] +mod tests { + use super::*; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::sync::mpsc; + + struct RecordingHandler { + requests: mpsc::UnboundedSender<(String, String)>, + } + + impl MetadataHandler for RecordingHandler { + async fn handle( + &self, + method: &str, + path: &str, + _request: &[u8], + stream: &mut S, + ) -> Result<()> { + self.requests + .send((method.to_string(), path.to_string())) + .unwrap(); + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") + .await + .map_err(|error| miette::miette!("{error}"))?; + Ok(()) + } + } + + async fn connection_pair() -> (tokio::net::TcpStream, tokio::net::TcpStream) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let client = tokio::net::TcpStream::connect(listener.local_addr().unwrap()) + .await + .unwrap(); + let (server, _) = listener.accept().await.unwrap(); + (client, server) + } + + #[tokio::test] + async fn phase0_metadata_loopback_dispatches_method_path_and_response() { + let (requests_tx, mut requests_rx) = mpsc::unbounded_channel(); + let handler = RecordingHandler { + requests: requests_tx, + }; + let (mut client, server) = connection_pair().await; + let server_task = tokio::spawn(async move { handle_connection(&handler, server).await }); + + client + .write_all(b"GET /computeMetadata/v1/instance HTTP/1.1\r\nHost: metadata\r\n\r\n") + .await + .unwrap(); + let mut response = Vec::new(); + client.read_to_end(&mut response).await.unwrap(); + server_task.await.unwrap().unwrap(); + + assert_eq!( + requests_rx.try_recv().unwrap(), + ( + "GET".to_string(), + "/computeMetadata/v1/instance".to_string() + ) + ); + assert_eq!(response, b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok"); + } + + #[tokio::test] + async fn phase0_metadata_loopback_rejects_oversized_headers_before_handler() { + let (requests_tx, mut requests_rx) = mpsc::unbounded_channel(); + let handler = RecordingHandler { + requests: requests_tx, + }; + let (mut client, server) = connection_pair().await; + let server_task = tokio::spawn(async move { handle_connection(&handler, server).await }); + + client + .write_all(&vec![b'x'; MAX_REQUEST_BYTES]) + .await + .unwrap(); + let mut response = Vec::new(); + client.read_to_end(&mut response).await.unwrap(); + server_task.await.unwrap().unwrap(); + + assert_eq!( + response, + b"HTTP/1.1 413 Request Entity Too Large\r\nContent-Length: 0\r\n\r\n" + ); + assert!(requests_rx.try_recv().is_err()); + } +} diff --git a/crates/openshell-supervisor-network/src/lib.rs b/crates/openshell-supervisor-network/src/lib.rs index ac0cb120a2..f5d0205e3a 100644 --- a/crates/openshell-supervisor-network/src/lib.rs +++ b/crates/openshell-supervisor-network/src/lib.rs @@ -20,3 +20,63 @@ pub mod sigv4; mod spiffe_endpoint; mod token_grant; pub mod upstream_proxy; + +#[cfg(test)] +pub(crate) mod test_alloc { + use std::alloc::{GlobalAlloc, Layout, System}; + use std::sync::atomic::{AtomicU64, Ordering}; + + struct CountingAllocator; + + static ALLOCATIONS: AtomicU64 = AtomicU64::new(0); + static ALLOCATED_BYTES: AtomicU64 = AtomicU64::new(0); + + #[allow(unsafe_code)] + unsafe impl GlobalAlloc for CountingAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let pointer = unsafe { System.alloc(layout) }; + if !pointer.is_null() { + ALLOCATIONS.fetch_add(1, Ordering::Relaxed); + ALLOCATED_BYTES.fetch_add(layout.size() as u64, Ordering::Relaxed); + } + pointer + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + let pointer = unsafe { System.alloc_zeroed(layout) }; + if !pointer.is_null() { + ALLOCATIONS.fetch_add(1, Ordering::Relaxed); + ALLOCATED_BYTES.fetch_add(layout.size() as u64, Ordering::Relaxed); + } + pointer + } + + unsafe fn dealloc(&self, pointer: *mut u8, layout: Layout) { + unsafe { System.dealloc(pointer, layout) }; + } + + unsafe fn realloc(&self, pointer: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + let pointer = unsafe { System.realloc(pointer, layout, new_size) }; + if !pointer.is_null() { + ALLOCATIONS.fetch_add(1, Ordering::Relaxed); + ALLOCATED_BYTES.fetch_add(new_size as u64, Ordering::Relaxed); + } + pointer + } + } + + #[global_allocator] + static GLOBAL: CountingAllocator = CountingAllocator; + + pub fn reset() { + ALLOCATIONS.store(0, Ordering::SeqCst); + ALLOCATED_BYTES.store(0, Ordering::SeqCst); + } + + pub fn snapshot() -> (u64, u64) { + ( + ALLOCATIONS.load(Ordering::SeqCst), + ALLOCATED_BYTES.load(Ordering::SeqCst), + ) + } +} diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index f0654c287d..2d945b1a34 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -125,6 +125,24 @@ pub struct OpaEngine { middleware_runner: RwLock, } +#[cfg(test)] +static TEST_OPA_QUERY_COUNT: AtomicU64 = AtomicU64::new(0); + +#[cfg(test)] +fn record_test_opa_query() { + TEST_OPA_QUERY_COUNT.fetch_add(1, Ordering::Relaxed); +} + +#[cfg(test)] +pub(crate) fn reset_test_opa_query_count() { + TEST_OPA_QUERY_COUNT.store(0, Ordering::SeqCst); +} + +#[cfg(test)] +pub(crate) fn test_opa_query_count() -> u64 { + TEST_OPA_QUERY_COUNT.load(Ordering::SeqCst) +} + /// Generation guard captured when an HTTP tunnel or request path starts. #[derive(Clone)] pub struct PolicyGenerationGuard { @@ -201,6 +219,15 @@ impl TunnelPolicyEngine { } impl OpaEngine { + #[cfg(test)] + pub(crate) fn poison_lock_for_test(&self) { + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _guard = self.engine.lock().expect("test engine lock"); + panic!("poison OPA engine lock for compatibility fallback test"); + })); + assert!(self.engine.is_poisoned()); + } + /// Load policy from a `.rego` rules file and data from a YAML file. /// /// Preprocesses the YAML data to expand access presets and validate L7 config. @@ -429,6 +456,9 @@ impl OpaEngine { &self, input: &NetworkInput, ) -> Result<(NetworkAction, u64)> { + #[cfg(test)] + record_test_opa_query(); + let input_json = network_input_json(input); let mut engine = self @@ -665,6 +695,9 @@ impl OpaEngine { &self, input: &NetworkInput, ) -> Result<(Vec, u64)> { + #[cfg(test)] + record_test_opa_query(); + let input_json = network_input_json(input); let mut engine = self @@ -723,6 +756,9 @@ impl OpaEngine { /// denial while preserving separate handling for `allowed_ips` and advisor /// proposals. pub fn query_exact_declared_endpoint_host(&self, input: &NetworkInput) -> Result { + #[cfg(test)] + record_test_opa_query(); + let input_json = network_input_json(input); let mut engine = self @@ -5003,6 +5039,7 @@ network_policies: port: 8567 protocol: rest enforcement: enforce + tls: skip allowed_ips: - 192.168.1.100 rules: @@ -5041,7 +5078,7 @@ process: } #[test] - fn overlapping_policies_endpoint_config_returns_result() { + fn phase0_overlapping_policy_outputs_are_snapshotted_independently() { let engine = OpaEngine::from_strings(TEST_POLICY, OVERLAPPING_L7_TEST_DATA) .expect("engine should load overlapping data"); let input = NetworkInput { @@ -5052,12 +5089,29 @@ process: ancestors: vec![], cmdline_paths: vec![], }; - // Should return config from one of the entries without error. - let config = engine.query_endpoint_config(&input).unwrap(); - assert!( - config.is_some(), - "Expected endpoint config for overlapping policies" + assert_eq!( + engine.evaluate_network_action(&input).unwrap(), + NetworkAction::Allow { + matched_policy: Some("allow_192_168_1_100_8567".to_string()) + } ); + + let (configs, generation) = engine + .query_endpoint_configs_with_generation(&input) + .unwrap(); + assert_eq!(generation, engine.current_generation()); + assert_eq!(configs.len(), 2); + assert_eq!(get_str(&configs[0], "tls").as_deref(), Some("skip")); + assert_eq!(get_str_array(&configs[0], "allowed_ips"), ["192.168.1.100"]); + assert_eq!(get_str(&configs[1], "tls"), None); + + let selected = engine.query_endpoint_config(&input).unwrap().unwrap(); + assert_eq!( + crate::l7::parse_tls_mode(&selected), + crate::l7::TlsMode::Skip + ); + assert_eq!(engine.query_allowed_ips(&input).unwrap(), ["192.168.1.100"]); + assert!(engine.query_exact_declared_endpoint_host(&input).unwrap()); } // ======================================================================== diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 6286e481e4..c2617d499a 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -807,6 +807,68 @@ fn emit_denial_simple( } } +#[allow(clippy::too_many_arguments)] +fn build_connect_allow_ocsf_event( + peer_addr: SocketAddr, + host: &str, + port: u16, + binary: &str, + pid: &str, + ancestors: &str, + cmdline: &str, + policy: &str, + l7_inspection: bool, +) -> openshell_ocsf::OcsfEvent { + let connect_msg = if l7_inspection { + "CONNECT_L7" + } else { + "CONNECT" + }; + NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Open) + .action(ActionId::Allowed) + .disposition(DispositionId::Allowed) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .dst_endpoint(Endpoint::from_domain(host, port)) + .src_endpoint_addr(peer_addr.ip(), peer_addr.port()) + .actor_process(Process::from_bypass(binary, pid, ancestors).with_cmd_line(cmdline)) + .firewall_rule(policy, "opa") + .message(format!("{connect_msg} allowed {host}:{port}")) + .build() +} + +#[allow(clippy::too_many_arguments)] +fn build_forward_allow_ocsf_event( + peer_addr: SocketAddr, + method: &str, + host: &str, + port: u16, + path: &str, + binary: &str, + pid: &str, + ancestors: &str, + cmdline: &str, + policy: &str, +) -> openshell_ocsf::OcsfEvent { + HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Other) + .action(ActionId::Allowed) + .disposition(DispositionId::Allowed) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .http_request(HttpRequest::new( + method, + OcsfUrl::new("http", host, path, port), + )) + .dst_endpoint(Endpoint::from_domain(host, port)) + .src_endpoint(Endpoint::from_ip(peer_addr.ip(), peer_addr.port())) + .actor_process(Process::from_bypass(binary, pid, ancestors).with_cmd_line(cmdline)) + .firewall_rule(policy, "opa") + .message(format!("FORWARD allowed {method} {host}:{port}{path}")) + .build() +} + #[allow(clippy::too_many_arguments)] async fn deny_connect_destination( client: &mut TcpStream, @@ -1313,29 +1375,17 @@ async fn handle_tcp_connection( // Log the allowed CONNECT — use CONNECT_L7 when L7 inspection follows, // so log consumers can distinguish L4-only decisions from tunnel lifecycle events. - let connect_msg = if should_inspect_l7 { - "CONNECT_L7" - } else { - "CONNECT" - }; - { - let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Open) - .action(ActionId::Allowed) - .disposition(DispositionId::Allowed) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .src_endpoint_addr(workload_addr.ip(), workload_addr.port()) - .actor_process( - Process::from_bypass(&binary_str, &pid_str, &ancestors_str) - .with_cmd_line(&cmdline_str), - ) - .firewall_rule(policy_str, "opa") - .message(format!("{connect_msg} allowed {host_lc}:{port}")) - .build(); - ocsf_emit!(event); - } + ocsf_emit!(build_connect_allow_ocsf_event( + workload_addr, + &host_lc, + port, + &binary_str, + &pid_str, + &ancestors_str, + &cmdline_str, + policy_str, + should_inspect_l7, + )); emit_connect_activity_if_l4_only(&activity_tx, l7_route); // `effective_tls_skip` was resolved before the `200` above (the fail-closed @@ -3530,9 +3580,13 @@ fn rewrite_forward_request( output.extend_from_slice(b"\r\n"); let rewritten_header_end = output.len(); - // Append any overflow body bytes from the original buffer + // Append only bytes that belong to the first request body. The initial + // proxy read can also contain a pipelined follow-on request; forwarding + // that as body overflow would bypass its own policy evaluation. if header_end < used { - output.extend_from_slice(&raw[header_end..used]); + let overflow = &raw[header_end..used]; + let body_prefix_len = initial_forward_body_prefix_len(&header_str, overflow); + output.extend_from_slice(&overflow[..body_prefix_len]); } // Fail-closed: scan for any remaining unresolved placeholders @@ -3553,6 +3607,66 @@ fn rewrite_forward_request( Ok(output) } +fn initial_forward_body_prefix_len(header_str: &str, overflow: &[u8]) -> usize { + match crate::l7::rest::parse_body_length(header_str) { + Ok(crate::l7::provider::BodyLength::None) => 0, + Ok(crate::l7::provider::BodyLength::ContentLength(len)) => usize::try_from(len) + .unwrap_or(usize::MAX) + .min(overflow.len()), + Ok(crate::l7::provider::BodyLength::Chunked) => { + complete_chunked_body_prefix_len(overflow).unwrap_or(overflow.len()) + } + // Invalid framing is rejected by the guarded relay before an upstream + // body write. Keep the bytes available so that parser sees the same + // malformed request instead of blocking while trying to re-read them. + Err(_) => overflow.len(), + } +} + +/// Return the complete chunked body length when its terminator is already in +/// the initial read. `None` means more body bytes are required. +fn complete_chunked_body_prefix_len(bytes: &[u8]) -> Option { + let mut pos = 0usize; + loop { + let line_end = bytes[pos..] + .windows(2) + .position(|window| window == b"\r\n")? + + pos; + let size_line = std::str::from_utf8(&bytes[pos..line_end]).ok()?; + let size = usize::from_str_radix( + size_line + .split(';') + .next() + .map(str::trim) + .unwrap_or_default(), + 16, + ) + .ok()?; + pos = line_end.checked_add(2)?; + + if size == 0 { + loop { + let trailer_end = bytes[pos..] + .windows(2) + .position(|window| window == b"\r\n")? + + pos; + let empty = trailer_end == pos; + pos = trailer_end.checked_add(2)?; + if empty { + return Some(pos); + } + } + } + + let chunk_end = pos.checked_add(size)?; + let framed_end = chunk_end.checked_add(2)?; + if framed_end > bytes.len() || &bytes[chunk_end..framed_end] != b"\r\n" { + return None; + } + pos = framed_end; + } +} + struct ForwardRelayOptions<'a> { generation_guard: &'a PolicyGenerationGuard, websocket_extensions: crate::l7::rest::WebSocketExtensionMode, @@ -4559,7 +4673,6 @@ async fn handle_forward_proxy( } }; } - forward_request_bytes = match inject_token_grant_for_forward_request( method, &upstream_target, @@ -10239,4 +10352,6 @@ network_policies: assert_eq!(res, 3); assert_eq!(unk, 2); } + #[path = "phase0.rs"] + mod phase0; } diff --git a/crates/openshell-supervisor-network/src/proxy/destination.rs b/crates/openshell-supervisor-network/src/proxy/destination.rs index 2653afd78f..c5cbc1776d 100644 --- a/crates/openshell-supervisor-network/src/proxy/destination.rs +++ b/crates/openshell-supervisor-network/src/proxy/destination.rs @@ -248,4 +248,56 @@ mod tests { assert_eq!(denial.kind, DestinationDenialKind::TrustedGateway); } + + #[test] + fn phase0_validation_mode_precedence_is_explicit_and_stable() { + let trusted_ip = IpAddr::V4(Ipv4Addr::new(169, 254, 1, 2)); + let trusted = build_validation_plan( + "host.openshell.internal", + "host.openshell.internal", + Some(trusted_ip), + &["10.0.0.0/8".to_string()], + true, + ) + .unwrap(); + assert_eq!( + trusted.address_authorization, + AddressAuthorization::TrustedGatewayAlias { + expected_ip: trusted_ip + } + ); + + let explicit = build_validation_plan( + "10.2.3.4", + "10.2.3.4", + None, + &["10.0.0.0/8".to_string()], + true, + ) + .unwrap(); + assert_eq!( + explicit.address_authorization, + AddressAuthorization::ExplicitAllowedIps(vec!["10.0.0.0/8".parse().unwrap()]) + ); + + let implicit = build_validation_plan("10.2.3.4", "10.2.3.4", None, &[], true).unwrap(); + assert_eq!( + implicit.address_authorization, + AddressAuthorization::ImplicitIpLiteral("10.2.3.4".parse().unwrap()) + ); + + let declared = + build_validation_plan("private.example", "private.example", None, &[], true).unwrap(); + assert_eq!( + declared.address_authorization, + AddressAuthorization::ExactDeclaredHost + ); + + let default = + build_validation_plan("*.example.com", "*.example.com", None, &[], false).unwrap(); + assert_eq!( + default.address_authorization, + AddressAuthorization::DefaultPublicOnly + ); + } } diff --git a/crates/openshell-supervisor-network/src/proxy/tests/phase0.rs b/crates/openshell-supervisor-network/src/proxy/tests/phase0.rs new file mode 100644 index 0000000000..a3f3260b9f --- /dev/null +++ b/crates/openshell-supervisor-network/src/proxy/tests/phase0.rs @@ -0,0 +1,652 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Compatibility fixtures for RFC 0005 Phase 0. + +use super::*; +use std::io::Write; +use std::sync::{Arc, Mutex}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tracing_subscriber::prelude::*; + +#[derive(Clone)] +struct SharedBuffer(Arc>>); + +impl Write for SharedBuffer { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +fn allowed_decision(intent: EgressIntent) -> EgressDecision { + EgressDecision { + intent, + action: NetworkAction::Allow { + matched_policy: Some("phase0".to_string()), + }, + l4_policy_generation: 0, + identity: ProcessIdentityEvidence::Available, + endpoint: EndpointDecision::default(), + binary: Some(PathBuf::from("/usr/bin/curl")), + binary_pid: Some(42), + ancestors: vec![PathBuf::from("/usr/bin/sh")], + cmdline_paths: vec![], + } +} + +async fn tcp_pair() -> (TcpStream, TcpStream) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let client = TcpStream::connect(listener.local_addr().unwrap()) + .await + .unwrap(); + let (server, _) = listener.accept().await.unwrap(); + (client, server) +} + +fn assert_json_response( + response: &[u8], + expected_status: &str, + expected_error: &str, + expected_detail: &str, +) { + let (headers, body) = response + .windows(4) + .position(|window| window == b"\r\n\r\n") + .map(|end| (&response[..end + 4], &response[end + 4..])) + .expect("complete HTTP response"); + let headers = String::from_utf8(headers.to_vec()).unwrap(); + assert!(headers.starts_with(expected_status)); + assert!(headers.contains("Content-Type: application/json\r\n")); + assert!(headers.contains(&format!("Content-Length: {}\r\n", body.len()))); + assert!(headers.contains("Connection: close\r\n")); + assert_eq!( + serde_json::from_slice::(body).unwrap(), + serde_json::json!({ + "error": expected_error, + "detail": expected_detail, + }) + ); +} + +#[tokio::test] +async fn destination_denials_preserve_adapter_specific_wire_contracts() { + let cases = [ + ( + DestinationDenialKind::TrustedGateway, + "trusted-gateway check failed", + ), + ( + DestinationDenialKind::InvalidAllowedIps, + "invalid allowed_ips in policy", + ), + ( + DestinationDenialKind::AllowedIps, + "allowed_ips check failed", + ), + ( + DestinationDenialKind::DeclaredEndpoint, + "declared endpoint check failed", + ), + (DestinationDenialKind::InternalAddress, "internal address"), + ]; + + for (kind, detail) in cases { + let denial = DestinationDenial { + kind, + reason: "phase0 destination failure".to_string(), + }; + let peer: SocketAddr = "127.0.0.1:41000".parse().unwrap(); + + let (mut app, mut proxy) = tcp_pair().await; + deny_connect_destination( + &mut proxy, + &denial, + peer, + "target.example", + 8443, + "/usr/bin/curl", + "42", + "/usr/bin/sh", + "curl", + &allowed_decision(EgressIntent::connect("target.example".to_string(), 8443)), + &None, + &None, + ) + .await + .unwrap(); + proxy.shutdown().await.unwrap(); + let mut response = Vec::new(); + app.read_to_end(&mut response).await.unwrap(); + assert_json_response( + &response, + "HTTP/1.1 403 Forbidden\r\n", + "ssrf_denied", + &format!("CONNECT target.example:8443 blocked: {detail}"), + ); + + let (mut app, mut proxy) = tcp_pair().await; + deny_forward_destination( + &mut proxy, + &denial, + peer, + "POST", + "target.example", + 8080, + "/v1/items", + "/usr/bin/curl", + "42", + "/usr/bin/sh", + "curl", + "phase0", + &allowed_decision(EgressIntent::forward_http( + "target.example".to_string(), + 8080, + )), + None, + None, + ) + .await + .unwrap(); + proxy.shutdown().await.unwrap(); + let mut response = Vec::new(); + app.read_to_end(&mut response).await.unwrap(); + assert_json_response( + &response, + "HTTP/1.1 403 Forbidden\r\n", + "ssrf_denied", + &format!("POST target.example:8080 blocked: {detail}"), + ); + } +} + +#[test] +fn representative_adapter_denials_preserve_ocsf_fields() { + let captured = Arc::new(Mutex::new(Vec::new())); + let layer = openshell_ocsf::tracing_layers::OcsfJsonlLayer::new(SharedBuffer(captured.clone())); + let subscriber = tracing_subscriber::registry().with(layer); + let denial_reason = "target.example resolves to internal address 10.0.0.5"; + tracing::subscriber::with_default(subscriber, || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async { + let denial = DestinationDenial { + kind: DestinationDenialKind::InternalAddress, + reason: denial_reason.to_string(), + }; + let peer: SocketAddr = "127.0.0.1:41000".parse().unwrap(); + + let (_app, mut proxy) = tcp_pair().await; + deny_connect_destination( + &mut proxy, + &denial, + peer, + "target.example", + 8443, + "/usr/bin/curl", + "42", + "/usr/bin/sh", + "curl --proxy", + &allowed_decision(EgressIntent::connect("target.example".to_string(), 8443)), + &None, + &None, + ) + .await + .unwrap(); + + let (_app, mut proxy) = tcp_pair().await; + deny_forward_destination( + &mut proxy, + &denial, + peer, + "POST", + "target.example", + 8080, + "/v1/items", + "/usr/bin/curl", + "42", + "/usr/bin/sh", + "curl --proxy", + "phase0", + &allowed_decision(EgressIntent::forward_http( + "target.example".to_string(), + 8080, + )), + None, + None, + ) + .await + .unwrap(); + }); + }); + + let bytes = captured.lock().unwrap().clone(); + let events: Vec = String::from_utf8(bytes) + .unwrap() + .lines() + .map(|line| serde_json::from_str(line).unwrap()) + .collect(); + assert_eq!(events.len(), 2); + + let connect = &events[0]; + assert_eq!(connect["class_name"], "Network Activity"); + assert_eq!(connect["activity_name"], "Open"); + assert_eq!(connect["action"], "Denied"); + assert_eq!(connect["disposition"], "Blocked"); + assert_eq!(connect["severity"], "Medium"); + assert_eq!(connect["status"], "Failure"); + assert_eq!(connect["dst_endpoint"]["domain"], "target.example"); + assert_eq!(connect["dst_endpoint"]["port"], 8443); + assert_eq!(connect["actor"]["process"]["name"], "/usr/bin/curl"); + assert_eq!(connect["actor"]["process"]["pid"], 42); + assert_eq!(connect["firewall_rule"]["name"], "-"); + assert_eq!(connect["firewall_rule"]["type"], "ssrf"); + assert_eq!( + connect["message"], + "CONNECT blocked: internal address target.example:8443" + ); + assert_eq!(connect["status_detail"], denial_reason); + + let forward = &events[1]; + assert_eq!(forward["class_name"], "HTTP Activity"); + assert_eq!(forward["activity_name"], "Other"); + assert_eq!(forward["action"], "Denied"); + assert_eq!(forward["disposition"], "Blocked"); + assert_eq!(forward["severity"], "Medium"); + assert_eq!(forward["status"], "Failure"); + assert_eq!(forward["dst_endpoint"]["domain"], "target.example"); + assert_eq!(forward["dst_endpoint"]["port"], 8080); + assert_eq!(forward["http_request"]["http_method"], "POST"); + assert_eq!(forward["firewall_rule"]["name"], "phase0"); + assert_eq!(forward["firewall_rule"]["type"], "ssrf"); + assert_eq!( + forward["message"], + "FORWARD blocked: internal IP without allowed_ips for target.example:8080" + ); + assert_eq!(forward["status_detail"], denial_reason); +} + +#[test] +fn representative_adapter_allows_preserve_ocsf_fields() { + let peer: SocketAddr = "127.0.0.1:41000".parse().unwrap(); + let connect = serde_json::to_value(build_connect_allow_ocsf_event( + peer, + "target.example", + 8443, + "/usr/bin/curl", + "42", + "/usr/bin/sh", + "curl --proxy", + "phase0", + true, + )) + .unwrap(); + assert_eq!(connect["class_name"], "Network Activity"); + assert_eq!(connect["activity_name"], "Open"); + assert_eq!(connect["action"], "Allowed"); + assert_eq!(connect["disposition"], "Allowed"); + assert_eq!(connect["severity"], "Informational"); + assert_eq!(connect["status"], "Success"); + assert_eq!(connect["dst_endpoint"]["domain"], "target.example"); + assert_eq!(connect["dst_endpoint"]["port"], 8443); + assert_eq!(connect["actor"]["process"]["name"], "/usr/bin/curl"); + assert_eq!(connect["firewall_rule"]["name"], "phase0"); + assert_eq!(connect["firewall_rule"]["type"], "opa"); + assert_eq!(connect["message"], "CONNECT_L7 allowed target.example:8443"); + + let forward = serde_json::to_value(build_forward_allow_ocsf_event( + peer, + "GET", + "target.example", + 8080, + "/v1/items", + "/usr/bin/curl", + "42", + "/usr/bin/sh", + "curl --proxy", + "phase0", + )) + .unwrap(); + assert_eq!(forward["class_name"], "HTTP Activity"); + assert_eq!(forward["activity_name"], "Other"); + assert_eq!(forward["action"], "Allowed"); + assert_eq!(forward["disposition"], "Allowed"); + assert_eq!(forward["severity"], "Informational"); + assert_eq!(forward["status"], "Success"); + assert_eq!(forward["dst_endpoint"]["domain"], "target.example"); + assert_eq!(forward["dst_endpoint"]["port"], 8080); + assert_eq!(forward["http_request"]["http_method"], "GET"); + assert_eq!(forward["firewall_rule"]["name"], "phase0"); + assert_eq!(forward["firewall_rule"]["type"], "opa"); + assert_eq!( + forward["message"], + "FORWARD allowed GET target.example:8080/v1/items" + ); +} + +fn poisoned_engine() -> OpaEngine { + let engine = OpaEngine::from_strings( + include_str!("../../../data/sandbox-policy.rego"), + r#" +network_policies: + phase0: + name: phase0 + endpoints: + - host: target.example + port: 443 + protocol: rest + enforcement: enforce + tls: skip + allowed_ips: ["10.0.0.0/8"] + rules: + - allow: { method: GET, path: "/**" } + binaries: + - path: /usr/bin/curl +"#, + ) + .unwrap(); + engine.poison_lock_for_test(); + engine +} + +#[test] +fn l7_query_failure_preserves_l4_only_fallback() { + let engine = poisoned_engine(); + let decision = allowed_decision(EgressIntent::connect("target.example".to_string(), 443)); + assert!(query_l7_route_snapshot(&engine, &decision, "target.example", 443).is_none()); +} + +#[test] +fn tls_query_failure_preserves_auto_fallback() { + let engine = poisoned_engine(); + let decision = allowed_decision(EgressIntent::connect("target.example".to_string(), 443)); + assert_eq!( + query_tls_mode(&engine, &decision, "target.example", 443), + crate::l7::TlsMode::Auto + ); +} + +#[test] +fn allowed_ips_query_failure_preserves_empty_fallback() { + let engine = poisoned_engine(); + let decision = allowed_decision(EgressIntent::connect("target.example".to_string(), 443)); + assert!(query_allowed_ips(&engine, &decision, "target.example", 443).is_empty()); +} + +#[test] +fn exact_host_query_failure_preserves_false_fallback() { + let engine = poisoned_engine(); + let decision = allowed_decision(EgressIntent::connect("target.example".to_string(), 443)); + assert!(!query_exact_declared_endpoint_host( + &engine, + &decision, + "target.example", + 443 + )); +} + +#[test] +fn identity_required_policy_accepts_real_binary_and_rejects_empty_exec_path() { + let engine = OpaEngine::from_strings( + include_str!("../../../data/sandbox-policy.rego"), + r#" +network_policies: + phase0: + name: phase0 + endpoints: + - host: target.example + port: 443 + binaries: + - path: /usr/bin/curl +"#, + ) + .unwrap(); + let input = |binary_path: PathBuf| crate::opa::NetworkInput { + host: "target.example".to_string(), + port: 443, + binary_path, + binary_sha256: String::new(), + ancestors: vec![], + cmdline_paths: vec![], + }; + + assert!(matches!( + engine + .evaluate_network_action(&input(PathBuf::from("/usr/bin/curl"))) + .unwrap(), + NetworkAction::Allow { .. } + )); + assert!(matches!( + engine + .evaluate_network_action(&input(PathBuf::new())) + .unwrap(), + NetworkAction::Deny { .. } + )); +} + +#[cfg(not(target_os = "linux"))] +#[test] +fn identity_required_mode_is_explicitly_unsupported_off_linux() { + let engine = OpaEngine::from_strings( + include_str!("../../../data/sandbox-policy.rego"), + "network_policies: {}\n", + ) + .unwrap(); + let decision = authorize_egress_intent( + "127.0.0.1:41000".parse().unwrap(), + &engine, + &BinaryIdentityCache::new(), + &AtomicU32::new(1), + EgressIntent::connect("target.example".to_string(), 443), + ); + + assert!(matches!(decision.action, NetworkAction::Deny { .. })); + assert_eq!( + decision.identity, + ProcessIdentityEvidence::Unavailable(IdentityUnavailableReason::UnsupportedPlatform) + ); +} + +#[test] +fn forward_rewrite_does_not_treat_a_pipelined_request_as_body_overflow() { + let raw = b"GET http://target.example/allowed HTTP/1.1\r\n\ + Host: target.example\r\n\ + Connection: keep-alive\r\n\r\n\ + POST http://target.example/blocked HTTP/1.1\r\n\ + Host: target.example\r\n\ + Content-Length: 0\r\n\r\n"; + let rewritten = rewrite_forward_request(raw, raw.len(), "/allowed", None, false).unwrap(); + let rewritten = String::from_utf8(rewritten).unwrap(); + + assert!(rewritten.starts_with("GET /allowed HTTP/1.1\r\n")); + assert!(rewritten.contains("Connection: close\r\n")); + assert!(!rewritten.contains("POST http://target.example/blocked")); +} + +#[test] +fn forward_rewrite_trims_pipeline_after_content_length_body() { + let raw = b"POST http://target.example/allowed HTTP/1.1\r\n\ + Host: target.example\r\n\ + Content-Length: 4\r\n\r\n\ + body\ + GET http://target.example/blocked HTTP/1.1\r\n\ + Host: target.example\r\n\r\n"; + let rewritten = rewrite_forward_request(raw, raw.len(), "/allowed", None, false).unwrap(); + let rewritten = String::from_utf8(rewritten).unwrap(); + + assert!(rewritten.ends_with("\r\n\r\nbody")); + assert!(!rewritten.contains("GET http://target.example/blocked")); +} + +#[test] +fn forward_rewrite_trims_pipeline_after_complete_chunked_body() { + let raw = b"POST http://target.example/allowed HTTP/1.1\r\n\ + Host: target.example\r\n\ + Transfer-Encoding: chunked\r\n\r\n\ + 4\r\nbody\r\n0\r\n\r\n\ + GET http://target.example/blocked HTTP/1.1\r\n\ + Host: target.example\r\n\r\n"; + let rewritten = rewrite_forward_request(raw, raw.len(), "/allowed", None, false).unwrap(); + let rewritten = String::from_utf8(rewritten).unwrap(); + + assert!(rewritten.ends_with("4\r\nbody\r\n0\r\n\r\n")); + assert!(!rewritten.contains("GET http://target.example/blocked")); +} + +#[tokio::test] +async fn forward_https_absolute_form_rejection_is_snapshotted() { + let (response, denial_stages) = drive_forward_through_handler( + " - { host: \"target.example\", port: 443 }\n", + "https://target.example/private", + ) + .await; + + assert_eq!( + response, + b"HTTP/1.1 400 Bad Request\r\nContent-Length: 27\r\n\r\nUse CONNECT for HTTPS URLs" + ); + assert!(denial_stages.is_empty()); +} + +async fn exercise_benchmark_request(proxy_addr: SocketAddr, target: SocketAddr, connect: bool) { + let mut client = TcpStream::connect(proxy_addr).await.unwrap(); + let authority = target.to_string(); + let request = if connect { + format!("CONNECT {authority} HTTP/1.1\r\nHost: {authority}\r\n\r\n") + } else { + format!( + "GET http://{authority}/phase0 HTTP/1.1\r\nHost: {authority}\r\nConnection: close\r\n\r\n" + ) + }; + client.write_all(request.as_bytes()).await.unwrap(); + let mut response = Vec::new(); + client.read_to_end(&mut response).await.unwrap(); + assert!(response.starts_with(b"HTTP/1.1 403 Forbidden")); +} + +/// Run with: +/// `cargo test -p openshell-supervisor-network phase0_proxy_performance_baseline -- --ignored --nocapture --test-threads=1` +#[test] +#[ignore = "manual Phase 0 allocation/query/latency baseline"] +fn phase0_proxy_performance_baseline() { + temp_env::with_vars( + [( + openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY, + Some("endpoint-only"), + )], + || { + tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap() + .block_on(async { + // Benchmark the full fail-closed path using a declared loopback + // destination. This is deterministic and never opens a listener + // outside the local process, so it does not trigger host firewall + // prompts during manual baseline collection. + let target: SocketAddr = "127.0.0.1:18080".parse().unwrap(); + + let policy = format!( + r#" +network_policies: + phase0: + name: phase0 + endpoints: + - host: {host} + port: {port} + tls: skip + binaries: + - path: "/**" +"#, + host = target.ip(), + port = target.port(), + ); + let engine = Arc::new( + OpaEngine::from_strings_with_binary_identity_required( + include_str!("../../../data/sandbox-policy.rego"), + &policy, + false, + ) + .unwrap(), + ); + let proxy_listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let proxy_addr = proxy_listener.local_addr().unwrap(); + let proxy_engine = engine.clone(); + let proxy_task = tokio::spawn(async move { + while let Ok((stream, _)) = proxy_listener.accept().await { + let engine = proxy_engine.clone(); + tokio::spawn(async move { + Box::pin(handle_tcp_connection( + stream, + engine, + Arc::new(BinaryIdentityCache::new()), + Arc::new(AtomicU32::new(0)), + None, + None, + None, + Arc::new(None), + None, + None, + None, + None, + )) + .await + .unwrap(); + }); + } + }); + + for connect in [true, false] { + exercise_benchmark_request(proxy_addr, target, connect).await; + } + + let iterations = std::env::var("OPENSHELL_PROXY_BASELINE_ITERATIONS") + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|value| *value > 0) + .unwrap_or(25); + let mut results = serde_json::Map::new(); + for (name, connect) in [("connect", true), ("forward", false)] { + crate::test_alloc::reset(); + crate::opa::reset_test_opa_query_count(); + let started = std::time::Instant::now(); + for _ in 0..iterations { + exercise_benchmark_request(proxy_addr, target, connect).await; + } + let elapsed = started.elapsed(); + let queries = crate::opa::test_opa_query_count(); + let (allocations, allocated_bytes) = crate::test_alloc::snapshot(); + let expected_queries = 4; + assert_eq!(queries, expected_queries * iterations); + results.insert( + name.to_string(), + serde_json::json!({ + "allocated_bytes_per_request": allocated_bytes / iterations, + "allocations_per_request": allocations / iterations, + "latency_ns_per_request": elapsed.as_nanos() / u128::from(iterations), + "opa_queries_per_request": queries / iterations, + }), + ); + } + println!( + "{}", + serde_json::json!({ + "iterations": iterations, + "phase0_proxy_baseline": results, + "scenario": "declared_loopback_destination_denied", + "schema_version": 1, + }) + ); + + proxy_task.abort(); + }); + }, + ); +} diff --git a/crates/openshell-supervisor-process/src/bypass_monitor/mod.rs b/crates/openshell-supervisor-process/src/bypass_monitor/mod.rs index 7d09d36f09..f04a2baefe 100644 --- a/crates/openshell-supervisor-process/src/bypass_monitor/mod.rs +++ b/crates/openshell-supervisor-process/src/bypass_monitor/mod.rs @@ -77,6 +77,62 @@ pub fn parse_kmsg_line(line: &str, namespace_prefix: &str) -> Option (openshell_ocsf::OcsfEvent, openshell_ocsf::OcsfEvent) { + let hint = hint_for_event(event); + let reason = "direct connection bypassed HTTP CONNECT proxy"; + let dst_port = event.dst_port.to_string(); + let dst_ep = if let Ok(ip) = event.dst_addr.parse::() { + Endpoint::from_ip(ip, event.dst_port) + } else { + Endpoint::from_domain(&event.dst_addr, event.dst_port) + }; + + let net_event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Refuse) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::Medium) + .dst_endpoint(dst_ep) + .actor_process(Process::from_bypass(binary, binary_pid, ancestors)) + .firewall_rule("bypass-detect", "nftables") + .observation_point(3) + .message(format!( + "BYPASS_DETECT {}:{} proto={} binary={binary} action=reject reason={reason}", + event.dst_addr, event.dst_port, event.proto, + )) + .build(); + + let finding_event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Open) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::Medium) + .is_alert(true) + .confidence(ConfidenceId::High) + .finding_info(FindingInfo::new("bypass-detect", "Proxy Bypass Detected").with_desc(reason)) + .remediation(hint) + .evidence_pairs(&[ + ("dst_addr", event.dst_addr.as_str()), + ("dst_port", dst_port.as_str()), + ("proto", event.proto.as_str()), + ("binary", binary), + ("binary_pid", binary_pid), + ("ancestors", ancestors), + ]) + .message(format!( + "BYPASS_DETECT {}:{} proto={} binary={binary} hint={hint}", + event.dst_addr, event.dst_port, event.proto, + )) + .build(); + + (net_event, finding_event) +} + /// Extract a single space-delimited field value from a nftables log line. /// /// Given `"DST="` and a string like `"...DST=93.184.216.34 LEN=60..."`, @@ -207,60 +263,11 @@ pub fn spawn( ("-".to_string(), "-".to_string(), "-".to_string()) }; - let hint = hint_for_event(&event); - let reason = "direct connection bypassed HTTP CONNECT proxy"; - // Dual-emit: Network Activity [4001] + Detection Finding [2004] - { - let dst_ep = if let Ok(ip) = event.dst_addr.parse::() { - Endpoint::from_ip(ip, event.dst_port) - } else { - Endpoint::from_domain(&event.dst_addr, event.dst_port) - }; - - let net_event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Refuse) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .dst_endpoint(dst_ep.clone()) - .actor_process(Process::from_bypass(&binary, &binary_pid, &ancestors)) - .firewall_rule("bypass-detect", "nftables") - .observation_point(3) - .message(format!( - "BYPASS_DETECT {}:{} proto={} binary={binary} action=reject reason={reason}", - event.dst_addr, event.dst_port, event.proto, - )) - .build(); - ocsf_emit!(net_event); - - let finding_event = DetectionFindingBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Open) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .is_alert(true) - .confidence(ConfidenceId::High) - .finding_info( - FindingInfo::new("bypass-detect", "Proxy Bypass Detected") - .with_desc(reason), - ) - .remediation(hint) - .evidence_pairs(&[ - ("dst_addr", &event.dst_addr), - ("dst_port", &event.dst_port.to_string()), - ("proto", &event.proto), - ("binary", &binary), - ("binary_pid", &binary_pid), - ("ancestors", &ancestors), - ]) - .message(format!( - "BYPASS_DETECT {}:{} proto={} binary={binary} hint={hint}", - event.dst_addr, event.dst_port, event.proto, - )) - .build(); - ocsf_emit!(finding_event); - } + let (net_event, finding_event) = + build_bypass_ocsf_events(&event, &binary, &binary_pid, &ancestors); + ocsf_emit!(net_event); + ocsf_emit!(finding_event); // Send to denial aggregator if available. if let Some(ref tx) = denial_tx { @@ -488,6 +495,49 @@ mod tests { assert!(hint_for_event(&event).contains("UDP")); } + #[test] + fn phase0_bypass_ocsf_contract_is_stable() { + let event = BypassEvent { + dst_addr: "93.184.216.34".to_string(), + dst_port: 443, + src_port: 48012, + proto: "tcp".to_string(), + uid: Some(1000), + }; + let (network, finding) = + build_bypass_ocsf_events(&event, "/usr/bin/curl", "42", "/usr/bin/sh"); + let network = serde_json::to_value(network).unwrap(); + assert_eq!(network["class_name"], "Network Activity"); + assert_eq!(network["activity_name"], "Refuse"); + assert_eq!(network["action"], "Denied"); + assert_eq!(network["disposition"], "Blocked"); + assert_eq!(network["severity"], "Medium"); + assert!(network.get("status").is_none()); + assert_eq!(network["dst_endpoint"]["ip"], "93.184.216.34"); + assert_eq!(network["dst_endpoint"]["port"], 443); + assert_eq!(network["actor"]["process"]["name"], "/usr/bin/curl"); + assert_eq!(network["firewall_rule"]["name"], "bypass-detect"); + assert_eq!(network["firewall_rule"]["type"], "nftables"); + assert_eq!(network["observation_point_id"], 3); + assert!( + network["message"] + .as_str() + .unwrap() + .contains("action=reject") + ); + + let finding = serde_json::to_value(finding).unwrap(); + assert_eq!(finding["class_name"], "Detection Finding"); + assert_eq!(finding["action"], "Denied"); + assert_eq!(finding["disposition"], "Blocked"); + assert_eq!(finding["severity"], "Medium"); + assert_eq!(finding["confidence"], "High"); + assert_eq!(finding["is_alert"], true); + assert_eq!(finding["finding_info"]["uid"], "bypass-detect"); + assert_eq!(finding["finding_info"]["title"], "Proxy Bypass Detected"); + assert_eq!(finding["evidences"][0]["data"]["dst_port"], "443"); + } + #[test] fn resolve_process_identity_surfaces_ambiguous_shared_socket() { use std::ffi::CString; diff --git a/e2e/rust/tests/proxy_egress_pipeline.rs b/e2e/rust/tests/proxy_egress_pipeline.rs index f226f1dd13..88b78ac181 100644 --- a/e2e/rust/tests/proxy_egress_pipeline.rs +++ b/e2e/rust/tests/proxy_egress_pipeline.rs @@ -16,7 +16,7 @@ use std::io::{self, Error, ErrorKind, Write}; use std::process::Stdio; -use std::sync::Mutex; +use std::sync::{Arc, Mutex}; use openshell_e2e::harness::binary::openshell_cmd; use openshell_e2e::harness::sandbox::SandboxGuard; @@ -165,6 +165,83 @@ network_policies: {} Ok(file) } +fn write_destination_denial_policy() -> Result { + let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; + let policy = r#"version: 1 + +filesystem_policy: + include_workdir: true + read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] + read_write: [/sandbox, /tmp, /dev/null] + +landlock: + compatibility: best_effort + +process: + run_as_user: sandbox + run_as_group: sandbox + +network_policies: + destination_denials: + name: destination_denials + endpoints: + - { host: 169.254.169.254, port: 80 } + - { host: 127.0.0.1, port: 80 } + - { host: 203.0.113.10, port: 6443 } + - host: 203.0.113.10 + port: 8080 + allowed_ips: ["198.51.100.0/24"] + binaries: + - path: "/**" +"#; + file.write_all(policy.as_bytes()) + .map_err(|error| format!("write policy: {error}"))?; + file.flush() + .map_err(|error| format!("flush policy: {error}"))?; + Ok(file) +} + +fn write_ip_literal_success_policy( + ip: &str, + explicit_port: u16, + implicit_port: u16, +) -> Result { + let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; + let policy = format!( + r#"version: 1 + +filesystem_policy: + include_workdir: true + read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] + read_write: [/sandbox, /tmp, /dev/null] + +landlock: + compatibility: best_effort + +process: + run_as_user: sandbox + run_as_group: sandbox + +network_policies: + destination_successes: + name: destination_successes + endpoints: + - host: {ip} + port: {explicit_port} + allowed_ips: ["{ip}/32"] + - host: {ip} + port: {implicit_port} + binaries: + - path: "/**" +"# + ); + file.write_all(policy.as_bytes()) + .map_err(|error| format!("write policy: {error}"))?; + file.flush() + .map_err(|error| format!("flush policy: {error}"))?; + Ok(file) +} + fn policy_path(file: &NamedTempFile) -> String { file.path() .to_str() @@ -282,6 +359,68 @@ struct EchoServer { task: JoinHandle<()>, } +struct PipelineProbeServer { + port: u16, + observed: Arc>>, + task: JoinHandle<()>, +} + +impl PipelineProbeServer { + async fn start() -> Result { + let listener = TcpListener::bind(("0.0.0.0", 0)) + .await + .map_err(|error| format!("bind pipeline probe: {error}"))?; + let port = listener + .local_addr() + .map_err(|error| format!("read pipeline probe address: {error}"))? + .port(); + let observed = Arc::new(Mutex::new(Vec::new())); + let observed_task = observed.clone(); + let task = tokio::spawn(async move { + while let Ok((mut stream, _)) = listener.accept().await { + let observed = observed_task.clone(); + tokio::spawn(async move { + let mut request = Vec::new(); + let mut buffer = [0_u8; 4096]; + loop { + match tokio::time::timeout( + std::time::Duration::from_millis(200), + stream.read(&mut buffer), + ) + .await + { + Ok(Ok(0)) | Err(_) => break, + Ok(Ok(read)) => request.extend_from_slice(&buffer[..read]), + Ok(Err(_)) => return, + } + } + observed.lock().unwrap().extend_from_slice(&request); + let _ = stream + .write_all( + b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok", + ) + .await; + }); + } + }); + Ok(Self { + port, + observed, + task, + }) + } + + fn observed_request(&self) -> Vec { + self.observed.lock().unwrap().clone() + } +} + +impl Drop for PipelineProbeServer { + fn drop(&mut self) { + self.task.abort(); + } +} + impl EchoServer { async fn start() -> Result { let listener = TcpListener::bind(("0.0.0.0", 0)) @@ -650,6 +789,180 @@ async fn policy_reload_updates_both_adapters_and_closes_existing_http_tunnel() { guard.cleanup().await; } +#[tokio::test] +async fn destination_denial_modes_match_across_connect_and_forward_adapters() { + let policy = write_destination_denial_policy().expect("write destination denial policy"); + let policy_path = policy_path(&policy); + let script = r#" +import json +import os +import socket +import urllib.parse + +proxy_url = next( + os.environ[name] + for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") + if os.environ.get(name) +) +parsed = urllib.parse.urlparse(proxy_url) + +def read_response(sock): + data = b"" + while b"\r\n\r\n" not in data: + chunk = sock.recv(4096) + if not chunk: + break + data += chunk + headers, _, body = data.partition(b"\r\n\r\n") + length = 0 + for line in headers.split(b"\r\n")[1:]: + if line.lower().startswith(b"content-length:"): + length = int(line.split(b":", 1)[1].strip()) + while len(body) < length: + chunk = sock.recv(4096) + if not chunk: + break + body += chunk + status = int(headers.split(None, 2)[1]) + return {"status": status, "body": json.loads(body.decode())} + +def connect_result(host, port): + with socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) as sock: + target = f"{host}:{port}" + sock.sendall(f"CONNECT {target} HTTP/1.1\r\nHost: {target}\r\n\r\n".encode()) + return read_response(sock) + +def forward_result(host, port): + with socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) as sock: + target = f"{host}:{port}" + sock.sendall( + f"GET http://{target}/probe HTTP/1.1\r\n" + f"Host: {target}\r\nConnection: close\r\n\r\n".encode() + ) + return read_response(sock) + +targets = { + "metadata": ("169.254.169.254", 80), + "loopback": ("127.0.0.1", 80), + "control_plane": ("203.0.113.10", 6443), + "outside_allowed_ips": ("203.0.113.10", 8080), +} +result = {} +for name, target in targets.items(): + result[name] = { + "connect": connect_result(*target), + "forward": forward_result(*target), + } +print(json.dumps(result, sort_keys=True)) +"#; + + let guard = SandboxGuard::create(&[ + "--policy", + &policy_path, + "--", + "python3", + "-c", + script, + ]) + .await + .expect("sandbox create"); + let result = parse_json_line(&guard.create_output); + for name in [ + "metadata", + "loopback", + "control_plane", + "outside_allowed_ips", + ] { + for adapter in ["connect", "forward"] { + assert_eq!( + result[name][adapter]["status"], 403, + "{name} {adapter}: {result}" + ); + assert_eq!( + result[name][adapter]["body"]["error"], "ssrf_denied", + "{name} {adapter}: {result}" + ); + } + } + assert_eq!( + result["metadata"]["connect"]["body"]["detail"], + "CONNECT 169.254.169.254:80 blocked: declared endpoint check failed" + ); + assert_eq!( + result["metadata"]["forward"]["body"]["detail"], + "GET 169.254.169.254:80 blocked: declared endpoint check failed" + ); + assert_eq!( + result["control_plane"]["connect"]["body"]["detail"], + "CONNECT 203.0.113.10:6443 blocked: allowed_ips check failed" + ); + assert_eq!( + result["outside_allowed_ips"]["forward"]["body"]["detail"], + "GET 203.0.113.10:8080 blocked: allowed_ips check failed" + ); +} + +#[tokio::test] +async fn explicit_allowed_ips_and_implicit_ip_literals_succeed_through_both_adapters() { + let resolver = SandboxGuard::create(&[ + "--", + "python3", + "-c", + "import socket; print('GATEWAY_IP=' + socket.gethostbyname('host.openshell.internal'))", + ]) + .await + .expect("resolve host gateway inside sandbox"); + let gateway_ip = resolver + .create_output + .lines() + .find_map(|line| line.trim().strip_prefix("GATEWAY_IP=")) + .expect("sandbox gateway IPv4 output") + .to_string(); + gateway_ip + .parse::() + .expect("host gateway must resolve to IPv4 for this Docker e2e"); + + let explicit_server = KeepAliveHttpServer::start() + .await + .expect("start explicit allowed_ips server"); + let implicit_server = KeepAliveHttpServer::start() + .await + .expect("start implicit IP-literal server"); + let policy = write_ip_literal_success_policy( + &gateway_ip, + explicit_server.port, + implicit_server.port, + ) + .expect("write IP literal policy"); + let policy_path = policy_path(&policy); + let mut guard = SandboxGuard::create_keep_with_args( + &["--policy", &policy_path], + &["sh", "-c", "echo Ready; sleep infinity"], + "Ready", + ) + .await + .expect("create keep sandbox"); + + for (mode, port) in [ + ("explicit_allowed_ips", explicit_server.port), + ("implicit_ip_literal", implicit_server.port), + ] { + let output = guard + .exec(&[ + "python3", + "-c", + &proxy_status_script(&gateway_ip, port), + ]) + .await + .unwrap_or_else(|error| panic!("exercise {mode}: {error}")); + let statuses = parse_json_line(&output); + assert_eq!(statuses["connect"], 200, "{mode} CONNECT: {statuses}"); + assert_eq!(statuses["forward"], 200, "{mode} forward: {statuses}"); + } + + guard.cleanup().await; +} + #[tokio::test] async fn tls_skip_connect_relays_opaque_bytes_bidirectionally() { let server = EchoServer::start().await.expect("start TCP echo server"); @@ -705,6 +1018,79 @@ print("RAW_RELAY_OK") ); } +#[tokio::test] +async fn forward_pipeline_never_reaches_upstream_as_first_request_overflow() { + let server = PipelineProbeServer::start() + .await + .expect("start pipeline probe server"); + let endpoint_options = r#" protocol: rest + enforcement: enforce + rules: + - allow: + method: GET + path: "/allowed""#; + let policy = write_policy(TEST_SERVER_HOST, server.port, endpoint_options) + .expect("write pipeline policy"); + let policy_path = policy_path(&policy); + let script = format!( + r#" +import os +import socket +import urllib.parse + +proxy_url = next( + os.environ[name] + for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") + if os.environ.get(name) +) +parsed = urllib.parse.urlparse(proxy_url) +target = "{host}:{port}" +first = ( + f"GET http://{{target}}/allowed HTTP/1.1\r\n" + f"Host: {{target}}\r\nConnection: keep-alive\r\n\r\n" +) +second = ( + f"POST http://{{target}}/blocked HTTP/1.1\r\n" + f"Host: {{target}}\r\nContent-Length: 0\r\n\r\n" +) +with socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) as sock: + sock.sendall((first + second).encode()) + response = b"" + while True: + chunk = sock.recv(4096) + if not chunk: + break + response += chunk +if response.count(b"HTTP/1.1 ") != 1 or b" 200 " not in response.split(b"\r\n", 1)[0]: + raise RuntimeError(f"unexpected pipelined response: {{response!r}}") +print("FORWARD_PIPELINE_CLOSED") +"#, + host = TEST_SERVER_HOST, + port = server.port, + ); + + let guard = SandboxGuard::create(&[ + "--policy", + &policy_path, + "--", + "python3", + "-c", + &script, + ]) + .await + .expect("sandbox create"); + assert!( + guard.create_output.contains("FORWARD_PIPELINE_CLOSED"), + "forward proxy did not close after one response:\n{}", + guard.create_output + ); + + let observed = String::from_utf8(server.observed_request()).expect("upstream HTTP request"); + assert!(observed.starts_with("GET /allowed HTTP/1.1\r\n")); + assert!(observed.contains("Connection: close\r\n")); + assert!(!observed.contains("/blocked")); +} + #[tokio::test] async fn http_credentials_are_rewritten_in_headers_and_bodies_for_both_adapters() { let _provider_lock = PROVIDER_LOCK diff --git a/e2e/rust/tests/websocket_conformance.rs b/e2e/rust/tests/websocket_conformance.rs index 65ba19aa1c..90f0e84024 100644 --- a/e2e/rust/tests/websocket_conformance.rs +++ b/e2e/rust/tests/websocket_conformance.rs @@ -373,7 +373,7 @@ def proxy_parts(): raise RuntimeError(f"invalid proxy URL: {{proxy_url!r}}") return parsed.hostname, parsed.port or 80 -def connect_with_retry(host, port, timeout_seconds=20): +def proxy_socket_with_retry(host, port, mode, timeout_seconds=20): proxy_host, proxy_port = proxy_parts() target = f"{{host}}:{{port}}" deadline = time.monotonic() + timeout_seconds @@ -382,13 +382,14 @@ def connect_with_retry(host, port, timeout_seconds=20): sock = None try: sock = socket.create_connection((proxy_host, proxy_port), timeout=5) - request = f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n" - sock.sendall(request.encode("ascii")) - response = recv_until(sock, b"\r\n\r\n").decode("iso-8859-1", "replace") - if response.startswith("HTTP/1.1 200") or response.startswith("HTTP/1.0 200"): - return sock - first_line = response.splitlines()[0] if response else "" - raise RuntimeError(f"proxy CONNECT failed: {{first_line}}") + if mode == "connect": + request = f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n" + sock.sendall(request.encode("ascii")) + response = recv_until(sock, b"\r\n\r\n").decode("iso-8859-1", "replace") + if not (response.startswith("HTTP/1.1 200") or response.startswith("HTTP/1.0 200")): + first_line = response.splitlines()[0] if response else "" + raise RuntimeError(f"proxy CONNECT failed: {{first_line}}") + return sock except (OSError, RuntimeError) as error: if sock is not None: sock.close() @@ -398,25 +399,28 @@ def connect_with_retry(host, port, timeout_seconds=20): token = os.environ[TOKEN_ENV] payload = json.dumps({{"authorization": "Bearer " + token}}, sort_keys=True) -key = base64.b64encode(os.urandom(16)).decode("ascii") - -with connect_with_retry(HOST, PORT) as sock: - request = ( - f"GET /ws HTTP/1.1\r\n" - f"Host: {{HOST}}:{{PORT}}\r\n" - "Upgrade: websocket\r\n" - "Connection: Upgrade\r\n" - f"Sec-WebSocket-Key: {{key}}\r\n" - "Sec-WebSocket-Version: 13\r\n" - "\r\n" - ) - sock.sendall(request.encode("ascii")) - response = recv_until(sock, b"\r\n\r\n").decode("iso-8859-1", "replace") - if not response.startswith("HTTP/1.1 101"): - raise RuntimeError("websocket upgrade failed") - sock.sendall(masked_text_frame(payload)) - _, response_payload = read_frame(sock) - print(response_payload.decode("utf-8")) +results = {{}} +for mode in ("connect", "forward"): + key = base64.b64encode(os.urandom(16)).decode("ascii") + with proxy_socket_with_retry(HOST, PORT, mode) as sock: + request_target = "/ws" if mode == "connect" else f"http://{{HOST}}:{{PORT}}/ws" + request = ( + f"GET {{request_target}} HTTP/1.1\r\n" + f"Host: {{HOST}}:{{PORT}}\r\n" + "Upgrade: websocket\r\n" + "Connection: Upgrade\r\n" + f"Sec-WebSocket-Key: {{key}}\r\n" + "Sec-WebSocket-Version: 13\r\n" + "\r\n" + ) + sock.sendall(request.encode("ascii")) + response = recv_until(sock, b"\r\n\r\n").decode("iso-8859-1", "replace") + if not response.startswith("HTTP/1.1 101"): + raise RuntimeError(f"{{mode}} websocket upgrade failed: {{response!r}}") + sock.sendall(masked_text_frame(payload)) + _, response_payload = read_frame(sock) + results[mode] = json.loads(response_payload.decode("utf-8")) +print(json.dumps(results, sort_keys=True)) "#, host = host, port = port, @@ -425,7 +429,7 @@ with connect_with_retry(HOST, PORT) as sock: } #[tokio::test] -async fn websocket_text_placeholder_is_rewritten_in_sandbox() { +async fn websocket_text_placeholder_is_rewritten_through_both_adapters() { let _provider_lock = PROVIDER_LOCK .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); @@ -465,7 +469,14 @@ async fn websocket_text_placeholder_is_rewritten_in_sandbox() { assert!( guard .create_output - .contains(r#"{"saw_placeholder": false, "saw_secret": true}"#), + .contains(r#""connect": {"saw_placeholder": false, "saw_secret": true}"#), + "expected CONNECT upstream to see only the resolved secret marker:\n{}", + guard.create_output + ); + assert!( + guard + .create_output + .contains(r#""forward": {"saw_placeholder": false, "saw_secret": true}"#), "expected upstream to see only the resolved secret marker:\n{}", guard.create_output ); From 1e05b26bfca0785c054eafda46c1e5be1627e1b4 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Mon, 20 Jul 2026 09:50:28 -0700 Subject: [PATCH 07/30] feat(policy): detect ambiguous network endpoints Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- crates/openshell-policy/src/ambiguity.rs | 590 +++++++++++++++++++++++ crates/openshell-policy/src/lib.rs | 4 + 2 files changed, 594 insertions(+) create mode 100644 crates/openshell-policy/src/ambiguity.rs diff --git a/crates/openshell-policy/src/ambiguity.rs b/crates/openshell-policy/src/ambiguity.rs new file mode 100644 index 0000000000..16b4e3653a --- /dev/null +++ b/crates/openshell-policy/src/ambiguity.rs @@ -0,0 +1,590 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Validation for endpoint selectors whose policy-derived behavior conflicts. + +use openshell_core::proto::{NetworkEndpoint, SandboxPolicy}; +use std::collections::{BTreeSet, HashSet, VecDeque}; +use std::fmt; + +/// One pair of endpoints that can authorize the same request but disagree on +/// policy-derived behavior that must have a single deterministic value. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EndpointAmbiguity { + pub left_policy: String, + pub left_endpoint_index: usize, + pub left_selector: String, + pub right_policy: String, + pub right_endpoint_index: usize, + pub right_selector: String, + pub overlapping_ports: Vec, + pub conflicts: Vec, +} + +impl fmt::Display for EndpointAmbiguity { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "network policies '{}' endpoint[{}] ({}) and '{}' endpoint[{}] ({}) overlap on port(s) {} with conflicting metadata: {}", + self.left_policy, + self.left_endpoint_index, + self.left_selector, + self.right_policy, + self.right_endpoint_index, + self.right_selector, + self.overlapping_ports + .iter() + .map(u32::to_string) + .collect::>() + .join(","), + self.conflicts.join("; "), + ) + } +} + +struct EndpointRef<'a> { + policy: &'a str, + index: usize, + endpoint: &'a NetworkEndpoint, +} + +/// Reject endpoint metadata ambiguity before a policy generation is activated. +/// +/// Request authorization rules (`access`, `rules`, and `deny_rules`) may be +/// contributed by multiple compatible endpoints. Metadata used to establish +/// or parse a connection must agree whenever the endpoint host, port, and (for +/// request-specific metadata) path selectors can match the same request. +#[must_use] +pub fn find_endpoint_ambiguities(policy: &SandboxPolicy) -> Vec { + let endpoints = policy + .network_policies + .iter() + .flat_map(|(key, rule)| { + let policy_name = if rule.name.is_empty() { + key.as_str() + } else { + rule.name.as_str() + }; + rule.endpoints + .iter() + .enumerate() + .map(move |(index, endpoint)| EndpointRef { + policy: policy_name, + index, + endpoint, + }) + }) + .collect::>(); + + let mut ambiguities = Vec::new(); + for left_index in 0..endpoints.len() { + for right_index in (left_index + 1)..endpoints.len() { + let left = &endpoints[left_index]; + let right = &endpoints[right_index]; + let overlapping_ports = overlapping_ports(left.endpoint, right.endpoint); + if overlapping_ports.is_empty() + || !host_patterns_overlap(&left.endpoint.host, &right.endpoint.host) + { + continue; + } + + let mut conflicts = connection_conflicts(left.endpoint, right.endpoint); + if path_patterns_overlap(&left.endpoint.path, &right.endpoint.path) { + conflicts.extend(request_pipeline_conflicts(left.endpoint, right.endpoint)); + } + if conflicts.is_empty() { + continue; + } + + ambiguities.push(EndpointAmbiguity { + left_policy: left.policy.to_string(), + left_endpoint_index: left.index, + left_selector: endpoint_selector(left.endpoint), + right_policy: right.policy.to_string(), + right_endpoint_index: right.index, + right_selector: endpoint_selector(right.endpoint), + overlapping_ports, + conflicts, + }); + } + } + ambiguities +} + +fn endpoint_selector(endpoint: &NetworkEndpoint) -> String { + let host = if endpoint.host.is_empty() { + "" + } else { + &endpoint.host + }; + let path = if endpoint.path.is_empty() { + "" + } else { + endpoint.path.as_str() + }; + format!("{host}:{}{}", display_ports(endpoint), path) +} + +fn display_ports(endpoint: &NetworkEndpoint) -> String { + effective_ports(endpoint) + .iter() + .map(u32::to_string) + .collect::>() + .join(",") +} + +fn effective_ports(endpoint: &NetworkEndpoint) -> BTreeSet { + if endpoint.ports.is_empty() { + (endpoint.port > 0) + .then_some(endpoint.port) + .into_iter() + .collect() + } else { + endpoint + .ports + .iter() + .copied() + .filter(|port| *port > 0) + .collect() + } +} + +fn overlapping_ports(left: &NetworkEndpoint, right: &NetworkEndpoint) -> Vec { + effective_ports(left) + .intersection(&effective_ports(right)) + .copied() + .collect() +} + +fn connection_conflicts(left: &NetworkEndpoint, right: &NetworkEndpoint) -> Vec { + let mut conflicts = Vec::new(); + push_conflict( + &mut conflicts, + "tls", + &normalized_tls(&left.tls), + &normalized_tls(&right.tls), + ); + push_conflict( + &mut conflicts, + "allowed_ips", + &normalized_strings(&left.allowed_ips), + &normalized_strings(&right.allowed_ips), + ); + push_conflict( + &mut conflicts, + "advisor_proposed", + &left.advisor_proposed, + &right.advisor_proposed, + ); + conflicts +} + +fn request_pipeline_conflicts(left: &NetworkEndpoint, right: &NetworkEndpoint) -> Vec { + let mut conflicts = Vec::new(); + push_conflict( + &mut conflicts, + "protocol", + &left.protocol.to_ascii_lowercase(), + &right.protocol.to_ascii_lowercase(), + ); + push_conflict( + &mut conflicts, + "enforcement", + &normalized_enforcement(&left.enforcement), + &normalized_enforcement(&right.enforcement), + ); + push_conflict( + &mut conflicts, + "allow_encoded_slash", + &left.allow_encoded_slash, + &right.allow_encoded_slash, + ); + push_conflict( + &mut conflicts, + "websocket_credential_rewrite", + &left.websocket_credential_rewrite, + &right.websocket_credential_rewrite, + ); + push_conflict( + &mut conflicts, + "request_body_credential_rewrite", + &left.request_body_credential_rewrite, + &right.request_body_credential_rewrite, + ); + push_conflict( + &mut conflicts, + "credential_signing", + &left.credential_signing, + &right.credential_signing, + ); + push_conflict( + &mut conflicts, + "signing_service", + &left.signing_service, + &right.signing_service, + ); + push_conflict( + &mut conflicts, + "signing_region", + &left.signing_region, + &right.signing_region, + ); + + if left.protocol.eq_ignore_ascii_case("graphql") + && right.protocol.eq_ignore_ascii_case("graphql") + { + push_conflict( + &mut conflicts, + "graphql_max_body_bytes", + &normalized_body_limit(left.graphql_max_body_bytes), + &normalized_body_limit(right.graphql_max_body_bytes), + ); + } + if is_json_rpc_family(&left.protocol) && is_json_rpc_family(&right.protocol) { + push_conflict( + &mut conflicts, + "json_rpc_max_body_bytes", + &normalized_body_limit(left.json_rpc_max_body_bytes), + &normalized_body_limit(right.json_rpc_max_body_bytes), + ); + } + if left.protocol.eq_ignore_ascii_case("mcp") && right.protocol.eq_ignore_ascii_case("mcp") { + push_conflict( + &mut conflicts, + "mcp.strict_tool_names", + &normalized_mcp_strict_tool_names(left), + &normalized_mcp_strict_tool_names(right), + ); + } + conflicts +} + +fn push_conflict( + conflicts: &mut Vec, + field: &str, + left: &T, + right: &T, +) { + if left != right { + conflicts.push(format!("{field}={left:?} vs {right:?}")); + } +} + +fn normalized_tls(value: &str) -> &'static str { + if value.eq_ignore_ascii_case("skip") { + "skip" + } else { + "auto" + } +} + +fn normalized_enforcement(value: &str) -> &'static str { + if value.eq_ignore_ascii_case("enforce") { + "enforce" + } else { + "audit" + } +} + +fn normalized_strings(values: &[String]) -> Vec { + values + .iter() + .map(|value| value.trim().to_ascii_lowercase()) + .collect::>() + .into_iter() + .collect() +} + +const DEFAULT_BODY_LIMIT: u32 = 65_536; + +fn normalized_body_limit(value: u32) -> u32 { + if value == 0 { + DEFAULT_BODY_LIMIT + } else { + value + } +} + +fn is_json_rpc_family(protocol: &str) -> bool { + protocol.eq_ignore_ascii_case("json-rpc") || protocol.eq_ignore_ascii_case("mcp") +} + +fn normalized_mcp_strict_tool_names(endpoint: &NetworkEndpoint) -> bool { + endpoint + .mcp + .as_ref() + .and_then(|options| options.strict_tool_names) + .unwrap_or(true) +} + +fn host_patterns_overlap(left: &str, right: &str) -> bool { + if left.is_empty() || right.is_empty() { + return true; + } + glob_patterns_overlap(&left.to_ascii_lowercase(), &right.to_ascii_lowercase(), '.') +} + +fn path_patterns_overlap(left: &str, right: &str) -> bool { + if left.is_empty() + || right.is_empty() + || matches!(left, "**" | "/**") + || matches!(right, "**" | "/**") + { + return true; + } + glob_patterns_overlap(left, right, '/') +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum GlobToken { + Literal(char), + Star { crosses_delimiter: bool }, +} + +fn tokenize_glob(pattern: &str) -> Vec { + let chars = pattern.chars().collect::>(); + let mut tokens = Vec::new(); + let mut index = 0; + while index < chars.len() { + if chars[index] != '*' { + tokens.push(GlobToken::Literal(chars[index])); + index += 1; + continue; + } + + let start = index; + while index < chars.len() && chars[index] == '*' { + index += 1; + } + tokens.push(GlobToken::Star { + crosses_delimiter: index - start >= 2, + }); + } + tokens +} + +/// Decide whether two delimiter-aware glob languages intersect. +/// +/// This is a small product-NFA search. `*` consumes any character except the +/// delimiter and `**` consumes any character, including the delimiter. Star +/// epsilon transitions and self-loops make the state space finite. +fn glob_patterns_overlap(left: &str, right: &str, delimiter: char) -> bool { + let left = tokenize_glob(left); + let right = tokenize_glob(right); + let mut queue = VecDeque::from([(0_usize, 0_usize)]); + let mut seen = HashSet::new(); + + while let Some((left_index, right_index)) = queue.pop_front() { + if !seen.insert((left_index, right_index)) { + continue; + } + if left_index == left.len() && right_index == right.len() { + return true; + } + + if matches!(left.get(left_index), Some(GlobToken::Star { .. })) { + queue.push_back((left_index + 1, right_index)); + } + if matches!(right.get(right_index), Some(GlobToken::Star { .. })) { + queue.push_back((left_index, right_index + 1)); + } + + let Some(left_token) = left.get(left_index) else { + continue; + }; + let Some(right_token) = right.get(right_index) else { + continue; + }; + if tokens_share_character(*left_token, *right_token, delimiter) { + let next_left = if matches!(left_token, GlobToken::Star { .. }) { + left_index + } else { + left_index + 1 + }; + let next_right = if matches!(right_token, GlobToken::Star { .. }) { + right_index + } else { + right_index + 1 + }; + queue.push_back((next_left, next_right)); + } + } + false +} + +fn tokens_share_character(left: GlobToken, right: GlobToken, delimiter: char) -> bool { + match (left, right) { + (GlobToken::Literal(left), GlobToken::Literal(right)) => left == right, + (GlobToken::Literal(value), GlobToken::Star { crosses_delimiter }) + | (GlobToken::Star { crosses_delimiter }, GlobToken::Literal(value)) => { + crosses_delimiter || value != delimiter + } + ( + GlobToken::Star { + crosses_delimiter: _, + }, + GlobToken::Star { + crosses_delimiter: _, + }, + ) => true, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use openshell_core::proto::{NetworkBinary, NetworkPolicyRule}; + + fn endpoint(host: &str, port: u32) -> NetworkEndpoint { + NetworkEndpoint { + host: host.to_string(), + port, + ..Default::default() + } + } + + fn policy_with(left: NetworkEndpoint, right: NetworkEndpoint) -> SandboxPolicy { + let mut policy = SandboxPolicy::default(); + policy.network_policies.insert( + "left".to_string(), + NetworkPolicyRule { + name: "left".to_string(), + endpoints: vec![left], + binaries: vec![NetworkBinary { + path: "/usr/bin/curl".to_string(), + ..Default::default() + }], + }, + ); + policy.network_policies.insert( + "right".to_string(), + NetworkPolicyRule { + name: "right".to_string(), + endpoints: vec![right], + binaries: vec![NetworkBinary { + path: "/usr/bin/bash".to_string(), + ..Default::default() + }], + }, + ); + policy + } + + #[test] + fn exact_and_wildcard_hosts_overlap() { + assert!(host_patterns_overlap("api.example.com", "*.example.com")); + assert!(host_patterns_overlap( + "us-aiplatform.googleapis.com", + "*-aiplatform.googleapis.com" + )); + assert!(!host_patterns_overlap("api.example.com", "*.other.com")); + } + + #[test] + fn intersecting_wildcards_are_detected() { + assert!(host_patterns_overlap("*.example.com", "api.*.com")); + assert!(host_patterns_overlap("**.example.com", "api.example.com")); + assert!(!host_patterns_overlap("*.example.com", "*.example.org")); + } + + #[test] + fn disjoint_ports_do_not_overlap() { + let mut left = endpoint("api.example.com", 443); + left.tls = "skip".to_string(); + let right = endpoint("api.example.com", 8443); + assert!(find_endpoint_ambiguities(&policy_with(left, right)).is_empty()); + } + + #[test] + fn compatible_request_rules_may_overlap() { + let mut left = endpoint("api.example.com", 443); + left.protocol = "rest".to_string(); + left.tls = "skip".to_string(); + let mut right = left.clone(); + left.access = "read-only".to_string(); + right.access = "read-write".to_string(); + + assert!(find_endpoint_ambiguities(&policy_with(left, right)).is_empty()); + } + + #[test] + fn disjoint_path_specific_protocols_may_overlap() { + let mut left = endpoint("api.example.com", 443); + left.path = "/graphql".to_string(); + left.protocol = "graphql".to_string(); + let mut right = endpoint("api.example.com", 443); + right.path = "/repos/**".to_string(); + right.protocol = "rest".to_string(); + + assert!(find_endpoint_ambiguities(&policy_with(left, right)).is_empty()); + } + + #[test] + fn exact_wildcard_tls_conflict_is_rejected() { + let mut left = endpoint("*.example.com", 443); + left.tls = "skip".to_string(); + let right = endpoint("api.example.com", 443); + let ambiguities = find_endpoint_ambiguities(&policy_with(left, right)); + + assert_eq!(ambiguities.len(), 1); + assert!(ambiguities[0].conflicts[0].contains("tls")); + assert!(ambiguities[0].to_string().contains("left")); + assert!(ambiguities[0].to_string().contains("right")); + } + + #[test] + fn allowed_ip_conflict_is_rejected_regardless_of_order() { + let mut left = endpoint("api.example.com", 443); + left.allowed_ips = vec!["10.0.1.0/24".to_string(), "10.0.0.0/24".to_string()]; + let mut compatible = endpoint("api.example.com", 443); + compatible.allowed_ips = vec!["10.0.0.0/24".to_string(), "10.0.1.0/24".to_string()]; + assert!(find_endpoint_ambiguities(&policy_with(left.clone(), compatible)).is_empty()); + + let mut conflicting = endpoint("api.example.com", 443); + conflicting.allowed_ips = vec!["10.0.2.0/24".to_string()]; + let ambiguities = find_endpoint_ambiguities(&policy_with(left, conflicting)); + assert!( + ambiguities[0] + .conflicts + .iter() + .any(|field| field.contains("allowed_ips")) + ); + } + + #[test] + fn credential_and_parser_conflicts_are_rejected_on_same_path() { + let mut left = endpoint("api.example.com", 443); + left.protocol = "rest".to_string(); + left.credential_signing = "sigv4".to_string(); + left.signing_service = "execute-api".to_string(); + let mut right = left.clone(); + right.signing_service = "bedrock".to_string(); + right.allow_encoded_slash = true; + + let ambiguities = find_endpoint_ambiguities(&policy_with(left, right)); + assert!( + ambiguities[0] + .conflicts + .iter() + .any(|field| field.contains("signing_service")) + ); + assert!( + ambiguities[0] + .conflicts + .iter() + .any(|field| field.contains("allow_encoded_slash")) + ); + } + + #[test] + fn different_binary_lists_do_not_hide_endpoint_ambiguity() { + let mut left = endpoint("api.example.com", 443); + left.tls = "skip".to_string(); + let right = endpoint("api.example.com", 443); + + assert_eq!( + find_endpoint_ambiguities(&policy_with(left, right)).len(), + 1 + ); + } +} diff --git a/crates/openshell-policy/src/lib.rs b/crates/openshell-policy/src/lib.rs index 7937a3757e..bbf7eadd62 100644 --- a/crates/openshell-policy/src/lib.rs +++ b/crates/openshell-policy/src/lib.rs @@ -18,6 +18,10 @@ use std::collections::{BTreeMap, HashMap}; use std::fmt; use std::path::Path; +mod ambiguity; + +pub use ambiguity::{EndpointAmbiguity, find_endpoint_ambiguities}; + use miette::{IntoDiagnostic, Result, WrapErr}; use openshell_core::proto::{ FilesystemPolicy, GraphqlOperation, L7Allow, L7DenyRule, L7QueryMatcher, L7Rule, From b751f4cf98d761688054b22a422b0d5123d3472c Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Mon, 20 Jul 2026 09:50:32 -0700 Subject: [PATCH 08/30] feat(sandbox): fail closed on invalid policy updates Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- crates/openshell-core/src/settings.rs | 42 ++ crates/openshell-sandbox/src/lib.rs | 550 +++++++++++++++++- crates/openshell-server/src/grpc/policy.rs | 22 + .../openshell-supervisor-network/src/opa.rs | 253 +++++++- 4 files changed, 826 insertions(+), 41 deletions(-) diff --git a/crates/openshell-core/src/settings.rs b/crates/openshell-core/src/settings.rs index 156e4c3845..4ae59272b0 100644 --- a/crates/openshell-core/src/settings.rs +++ b/crates/openshell-core/src/settings.rs @@ -107,6 +107,16 @@ pub const PROPOSAL_APPROVAL_MODE_KEY: &str = "proposal_approval_mode"; /// fail-closes on unknown persisted values for defense in depth. pub const PROPOSAL_APPROVAL_MODE_VALUES: &[&str] = &["manual", "auto"]; +/// Gateway security posture when a sandbox rejects a candidate policy during +/// validation. The default is [`POLICY_VALIDATION_FAILURE_MODE_FAIL_CLOSED`]. +pub const POLICY_VALIDATION_FAILURE_MODE_KEY: &str = "policy_validation_failure_mode"; +pub const POLICY_VALIDATION_FAILURE_MODE_FAIL_CLOSED: &str = "fail_closed"; +pub const POLICY_VALIDATION_FAILURE_MODE_RETAIN_LAST_VALID: &str = "retain_last_valid"; +pub const POLICY_VALIDATION_FAILURE_MODE_VALUES: &[&str] = &[ + POLICY_VALIDATION_FAILURE_MODE_FAIL_CLOSED, + POLICY_VALIDATION_FAILURE_MODE_RETAIN_LAST_VALID, +]; + pub const REGISTERED_SETTINGS: &[RegisteredSetting] = &[ // Gateway-level opt-in for provider profile policy composition. Defaults // to false when unset. @@ -137,6 +147,14 @@ pub const REGISTERED_SETTINGS: &[RegisteredSetting] = &[ kind: SettingValueKind::String, allowed_string_values: Some(PROPOSAL_APPROVAL_MODE_VALUES), }, + // Gateway security posture for policy validation failures. Defaults to + // fail_closed when unset. Operators may explicitly select availability- + // oriented last-known-good retention. + RegisteredSetting { + key: POLICY_VALIDATION_FAILURE_MODE_KEY, + kind: SettingValueKind::String, + allowed_string_values: Some(POLICY_VALIDATION_FAILURE_MODE_VALUES), + }, ]; /// Resolve a setting descriptor from the registry by key. @@ -168,6 +186,8 @@ pub fn parse_bool_like(raw: &str) -> Option { #[cfg(test)] mod tests { use super::{ + POLICY_VALIDATION_FAILURE_MODE_FAIL_CLOSED, POLICY_VALIDATION_FAILURE_MODE_KEY, + POLICY_VALIDATION_FAILURE_MODE_RETAIN_LAST_VALID, POLICY_VALIDATION_FAILURE_MODE_VALUES, PROPOSAL_APPROVAL_MODE_KEY, PROPOSAL_APPROVAL_MODE_VALUES, PROVIDERS_V2_ENABLED_KEY, REGISTERED_SETTINGS, RegisteredSetting, SettingValueKind, parse_bool_like, registered_keys_csv, setting_for_key, @@ -237,6 +257,28 @@ mod tests { } } + #[test] + fn policy_validation_failure_mode_accepts_only_documented_values() { + let setting = setting_for_key(POLICY_VALIDATION_FAILURE_MODE_KEY) + .expect("policy validation failure mode should be registered"); + assert_eq!(setting.kind, SettingValueKind::String); + assert_eq!( + setting.allowed_string_values, + Some(POLICY_VALIDATION_FAILURE_MODE_VALUES) + ); + assert!( + setting + .validate_string_value(POLICY_VALIDATION_FAILURE_MODE_FAIL_CLOSED) + .is_ok() + ); + assert!( + setting + .validate_string_value(POLICY_VALIDATION_FAILURE_MODE_RETAIN_LAST_VALID) + .is_ok() + ); + assert!(setting.validate_string_value("keep_old").is_err()); + } + // ---- parse_bool_like ---- #[test] diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 5ba41b9d7e..e55014a98e 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -25,7 +25,8 @@ use tracing::{debug, info, warn}; use openshell_ocsf::{ ActionId, ActivityId, AppLifecycleBuilder, ConfigStateChangeBuilder, DetectionFindingBuilder, - DispositionId, FindingInfo, SandboxContext, SeverityId, StateId, StatusId, ocsf_emit, + DispositionId, FindingInfo, OcsfEvent, SandboxContext, SeverityId, StateId, StatusId, + ocsf_emit, }; // --------------------------------------------------------------------------- @@ -852,7 +853,10 @@ fn load_policy_from_sidecar_bootstrap( policy, opa_engine, Some(proto), - LoadedPolicyOrigin::Gateway { revision: None }, + LoadedPolicyOrigin::Gateway { + revision: None, + has_last_valid_policy: true, + }, )) } @@ -1992,7 +1996,7 @@ async fn load_policy( } } - let loaded_policy_revision = + let mut loaded_policy_revision = policy_bound_to_snapshot.then(|| LoadedPolicyRevision::from_snapshot(&snapshot)); // Build OPA engine from baked-in rules + typed proto data. @@ -2002,12 +2006,37 @@ async fn load_policy( // container hasn't started yet. After the entrypoint spawns, the // engine is rebuilt with the real PID for symlink resolution. info!("Creating OPA engine from proto policy data"); + let mut has_last_valid_policy = true; let engine = match OpaEngine::from_proto(&proto_policy) { - Ok(engine) => engine, + Ok(engine) => Arc::new(engine), Err(e) => { report_initial_policy_failure(endpoint, id, loaded_policy_revision.as_ref(), &e) .await; - return Err(e); + let validation_error = e.to_string(); + let candidate_version = snapshot.version; + let candidate_hash = snapshot.policy_hash.clone(); + // There is no in-memory last-known-good generation during + // startup, so both configured modes necessarily fail closed. + // Load the restrictive default atomically and keep the + // rejected revision unacknowledged for poll reconciliation. + has_last_valid_policy = false; + proto_policy = openshell_policy::restrictive_default_policy(); + let engine = Arc::new(OpaEngine::from_proto(&proto_policy)?); + let disposition = apply_policy_validation_failure( + &engine, + PolicyValidationFailureMode::from_settings(&snapshot.settings), + has_last_valid_policy, + candidate_version, + &validation_error, + )?; + emit_policy_validation_failure( + &disposition, + candidate_version, + &candidate_hash, + &validation_error, + ); + loaded_policy_revision = None; + engine } }; @@ -2050,7 +2079,7 @@ async fn load_policy( } else { MiddlewareRegistryStatus::Synchronized }; - let opa_engine = Some(Arc::new(engine)); + let opa_engine = Some(engine); let policy = match SandboxPolicy::try_from(proto_policy.clone()) { Ok(policy) => policy, @@ -2067,6 +2096,7 @@ async fn load_policy( middleware_registry_status, LoadedPolicyOrigin::Gateway { revision: loaded_policy_revision, + has_last_valid_policy, }, agent_proposals_enabled_from_settings(&snapshot.settings), )); @@ -2249,6 +2279,7 @@ enum LoadedPolicyOrigin { LocalOverride, Gateway { revision: Option, + has_last_valid_policy: bool, }, } @@ -2256,6 +2287,16 @@ impl LoadedPolicyOrigin { fn allows_gateway_policy_reload(&self) -> bool { matches!(self, Self::Gateway { .. }) } + + fn has_last_valid_policy(&self) -> bool { + match self { + Self::LocalOverride => true, + Self::Gateway { + has_last_valid_policy, + .. + } => *has_last_valid_policy, + } + } } impl LoadedPolicyRevision { @@ -2362,7 +2403,7 @@ fn initial_poll_disposition( ) -> InitialPollDisposition { match origin { LoadedPolicyOrigin::LocalOverride => InitialPollDisposition::TrackOnly, - LoadedPolicyOrigin::Gateway { revision } => { + LoadedPolicyOrigin::Gateway { revision, .. } => { initial_policy_ack_candidate(revision.as_ref(), canonical).map_or( InitialPollDisposition::Reconcile, InitialPollDisposition::Acknowledge, @@ -2594,6 +2635,183 @@ async fn reconcile_middleware_registry( } } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum PolicyValidationFailureMode { + FailClosed, + RetainLastValid, +} + +impl PolicyValidationFailureMode { + fn from_settings( + settings: &std::collections::HashMap, + ) -> Self { + match extract_string_setting( + settings, + openshell_core::settings::POLICY_VALIDATION_FAILURE_MODE_KEY, + ) { + Some(openshell_core::settings::POLICY_VALIDATION_FAILURE_MODE_RETAIN_LAST_VALID) => { + Self::RetainLastValid + } + _ => Self::FailClosed, + } + } + + const fn as_str(self) -> &'static str { + match self { + Self::FailClosed => { + openshell_core::settings::POLICY_VALIDATION_FAILURE_MODE_FAIL_CLOSED + } + Self::RetainLastValid => { + openshell_core::settings::POLICY_VALIDATION_FAILURE_MODE_RETAIN_LAST_VALID + } + } + } +} + +#[derive(Debug, PartialEq, Eq)] +struct PolicyValidationFailureDisposition { + configured_mode: PolicyValidationFailureMode, + mode: PolicyValidationFailureMode, + previous_policy_active: bool, + active_generation: u64, +} + +struct RejectedPolicyGeneration { + version: u32, + policy_hash: String, + validation_error: String, + configured_mode: PolicyValidationFailureMode, +} + +fn apply_policy_validation_failure( + engine: &OpaEngine, + configured_mode: PolicyValidationFailureMode, + has_last_valid_policy: bool, + version: u32, + error: &str, +) -> Result { + let mode = if has_last_valid_policy { + configured_mode + } else { + PolicyValidationFailureMode::FailClosed + }; + match mode { + PolicyValidationFailureMode::FailClosed => { + let reason = format!( + "policy validation failed; fail-closed quarantine is active; candidate version {version} rejected: {error}" + ); + let active_generation = engine.enter_fail_closed(reason)?; + Ok(PolicyValidationFailureDisposition { + configured_mode, + mode, + previous_policy_active: false, + active_generation, + }) + } + PolicyValidationFailureMode::RetainLastValid => { + let active_generation = engine.exit_fail_closed()?; + Ok(PolicyValidationFailureDisposition { + configured_mode, + mode, + previous_policy_active: true, + active_generation, + }) + } + } +} + +fn policy_validation_failure_events( + disposition: &PolicyValidationFailureDisposition, + version: u32, + policy_hash: &str, + error: &str, +) -> [OcsfEvent; 2] { + let previous_policy_state = if disposition.previous_policy_active { + "IS active" + } else { + "IS NOT active" + }; + let state = if disposition.previous_policy_active { + (StateId::Enabled, "retained_last_valid") + } else { + (StateId::Disabled, "fail_closed") + }; + let message = format!( + "Policy validation failed; configured_mode={} effective_mode={}; previous policy {previous_policy_state} [version:{version} active_generation:{} error:{error}]", + disposition.configured_mode.as_str(), + disposition.mode.as_str(), + disposition.active_generation, + ); + let finding_uid = format!("policy-validation-failed-{version}"); + let version_string = version.to_string(); + let config = ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::High) + .status(StatusId::Failure) + .state(state.0, state.1) + .unmapped("candidate_version", serde_json::json!(version)) + .unmapped("candidate_policy_hash", serde_json::json!(policy_hash)) + .unmapped( + "validation_failure_mode", + serde_json::json!(disposition.mode.as_str()), + ) + .unmapped( + "configured_validation_failure_mode", + serde_json::json!(disposition.configured_mode.as_str()), + ) + .unmapped( + "previous_policy_active", + serde_json::json!(disposition.previous_policy_active), + ) + .unmapped( + "active_generation", + serde_json::json!(disposition.active_generation), + ) + .unmapped("validation_error", serde_json::json!(error)) + .message(message.clone()) + .build(); + let finding = DetectionFindingBuilder::new(ocsf_ctx()) + .activity(ActivityId::Open) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::High) + .is_alert(true) + .finding_info( + FindingInfo::new(&finding_uid, "Invalid policy generation rejected").with_desc(error), + ) + .evidence_pairs(&[ + ("candidate_version", &version_string), + ("candidate_policy_hash", policy_hash), + ("validation_failure_mode", disposition.mode.as_str()), + ( + "configured_validation_failure_mode", + disposition.configured_mode.as_str(), + ), + ( + "previous_policy_active", + if disposition.previous_policy_active { + "true" + } else { + "false" + }, + ), + ]) + .remediation("Submit a valid, unambiguous policy generation") + .message(message) + .build(); + [config, finding] +} + +fn emit_policy_validation_failure( + disposition: &PolicyValidationFailureDisposition, + version: u32, + policy_hash: &str, + error: &str, +) { + for event in policy_validation_failure_events(disposition, version, policy_hash, error) { + ocsf_emit!(event); + } +} + async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { use openshell_core::grpc_client::CachedOpenShellClient; use openshell_core::proto::PolicySource; @@ -2618,6 +2836,8 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { > = std::collections::HashMap::new(); let reloads_gateway_policy = ctx.loaded_policy_origin.allows_gateway_policy_reload(); let mut last_failed_runtime_revision: Option<(u64, String)> = None; + let mut rejected_policy_generation: Option = None; + let mut has_last_valid_policy = ctx.loaded_policy_origin.has_last_valid_policy(); // A first poll that does not match the policy already loaded into OPA must // pass through the normal reconciliation path immediately. It must never @@ -2707,14 +2927,23 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { ¤t_middleware_services, &result.supervisor_middleware_services, ); - let policy_runtime_changed = gateway_policy_runtime_needs_reconciliation( - reloads_gateway_policy, - ¤t_policy_hash, - &result.policy_hash, - ¤t_middleware_services, - &result.supervisor_middleware_services, - middleware_registry_status, - ); + // A valid candidate may intentionally restore byte-for-byte policy + // content that was active before a rejected update. Its hash then + // equals `current_policy_hash`, but the runtime is still quarantined + // and must reload (or it would remain deny-all indefinitely). + let recovering_rejected_policy = reloads_gateway_policy + && rejected_policy_generation + .as_ref() + .is_some_and(|rejected| rejected.policy_hash != result.policy_hash); + let policy_runtime_changed = recovering_rejected_policy + || gateway_policy_runtime_needs_reconciliation( + reloads_gateway_policy, + ¤t_policy_hash, + &result.policy_hash, + ¤t_middleware_services, + &result.supervisor_middleware_services, + middleware_registry_status, + ); // A local policy override is not coupled to the gateway policy // snapshot, so its service registry can still be reconciled alone. @@ -2738,6 +2967,30 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { // Log which settings changed. log_setting_changes(¤t_settings, &result.settings); + // A posture change after a rejected update takes effect immediately. + // The compiled last-known-good engine remains available beneath a + // fail-closed quarantine, so an explicit retain_last_valid selection + // can reactivate it without accepting any part of the invalid policy. + if !policy_changed && let Some(rejected) = rejected_policy_generation.as_mut() { + let mode = PolicyValidationFailureMode::from_settings(&result.settings); + if mode != rejected.configured_mode { + let disposition = apply_policy_validation_failure( + &ctx.opa_engine, + mode, + has_last_valid_policy, + rejected.version, + &rejected.validation_error, + )?; + emit_policy_validation_failure( + &disposition, + rejected.version, + &rejected.policy_hash, + &rejected.validation_error, + ); + rejected.configured_mode = mode; + } + } + ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) .severity(SeverityId::Informational) .status(StatusId::Success) @@ -2831,6 +3084,8 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { .policy .as_ref() .expect("successful runtime reload requires a policy payload"); + has_last_valid_policy = true; + rejected_policy_generation = None; if policy_changed { if let Some(policy_local_ctx) = ctx.policy_local_ctx.as_ref() { policy_local_ctx.set_current_policy(policy.clone()).await; @@ -2875,6 +3130,26 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { PolicyStatusUpdate::loaded(result.version), ); } + } else if recovering_rejected_policy + && result.version > 0 + && result.policy_source == PolicySource::Sandbox + { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "loaded") + .unmapped("policy_hash", serde_json::json!(&result.policy_hash)) + .message(format!( + "Policy reloaded successfully and fail-closed quarantine cleared [policy_hash:{}]", + result.policy_hash + )) + .build() + ); + enqueue_policy_status( + &status_sender, + PolicyStatusUpdate::loaded(result.version), + ); } if middleware_registry_changed { @@ -2901,6 +3176,7 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { Err(e) => { let failed_revision = (result.config_revision, result.policy_hash.clone()); if last_failed_runtime_revision.as_ref() != Some(&failed_revision) { + let validation_error = e.to_string(); ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) .severity(SeverityId::Medium) .status(StatusId::Failure) @@ -2912,13 +3188,35 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { result.version )) .build()); + + let failure_mode = + PolicyValidationFailureMode::from_settings(&result.settings); + let disposition = apply_policy_validation_failure( + &ctx.opa_engine, + failure_mode, + has_last_valid_policy, + result.version, + &validation_error, + )?; + emit_policy_validation_failure( + &disposition, + result.version, + &result.policy_hash, + &validation_error, + ); + rejected_policy_generation = Some(RejectedPolicyGeneration { + version: result.version, + policy_hash: result.policy_hash.clone(), + validation_error: validation_error.clone(), + configured_mode: failure_mode, + }); if policy_changed && result.version > 0 && result.policy_source == PolicySource::Sandbox { enqueue_policy_status( &status_sender, - PolicyStatusUpdate::failed(result.version, e.to_string()), + PolicyStatusUpdate::failed(result.version, validation_error), ); } } @@ -3033,6 +3331,22 @@ fn apply_agent_proposals_enabled( } } +/// Extract a string value from an effective setting, if present. +fn extract_string_setting<'a>( + settings: &'a std::collections::HashMap, + key: &str, +) -> Option<&'a str> { + use openshell_core::proto::setting_value; + settings + .get(key) + .and_then(|es| es.value.as_ref()) + .and_then(|sv| sv.value.as_ref()) + .and_then(|value| match value { + setting_value::Value::StringValue(value) => Some(value.as_str()), + _ => None, + }) +} + /// Log individual setting changes between two snapshots. fn log_setting_changes( old: &std::collections::HashMap, @@ -3684,6 +3998,7 @@ filesystem_policy: initial_poll_disposition( &LoadedPolicyOrigin::Gateway { revision: Some(loaded), + has_last_valid_policy: true, }, &canonical, ), @@ -3713,7 +4028,10 @@ filesystem_policy: 2, openshell_core::proto::PolicySource::Sandbox, ); - let origin = LoadedPolicyOrigin::Gateway { revision: None }; + let origin = LoadedPolicyOrigin::Gateway { + revision: None, + has_last_valid_policy: true, + }; assert_eq!( initial_poll_disposition(&origin, &canonical), @@ -3753,4 +4071,202 @@ filesystem_policy: "workspace must survive the snapshot so sync_policy_and_fetch_snapshot receives it" ); } + + fn string_effective_setting(value: &str) -> openshell_core::proto::EffectiveSetting { + openshell_core::proto::EffectiveSetting { + value: Some(openshell_core::proto::SettingValue { + value: Some(openshell_core::proto::setting_value::Value::StringValue( + value.to_string(), + )), + }), + scope: openshell_core::proto::SettingScope::Global.into(), + } + } + + #[test] + fn policy_validation_failure_mode_defaults_to_fail_closed() { + let settings = std::collections::HashMap::new(); + assert_eq!( + PolicyValidationFailureMode::from_settings(&settings), + PolicyValidationFailureMode::FailClosed + ); + } + + #[test] + fn policy_validation_failure_mode_requires_explicit_retain_setting() { + let settings = std::collections::HashMap::from([( + openshell_core::settings::POLICY_VALIDATION_FAILURE_MODE_KEY.to_string(), + string_effective_setting( + openshell_core::settings::POLICY_VALIDATION_FAILURE_MODE_RETAIN_LAST_VALID, + ), + )]); + assert_eq!( + PolicyValidationFailureMode::from_settings(&settings), + PolicyValidationFailureMode::RetainLastValid + ); + } + + #[test] + fn fail_closed_validation_failure_deactivates_previous_generation() { + let engine = OpaEngine::from_strings( + include_str!("../../openshell-supervisor-network/data/sandbox-policy.rego"), + "network_policies: {}\n", + ) + .unwrap(); + let previous_generation = engine.current_generation(); + + let disposition = apply_policy_validation_failure( + &engine, + PolicyValidationFailureMode::FailClosed, + true, + 7, + "conflicting tls metadata", + ) + .unwrap(); + + assert!(!disposition.previous_policy_active); + assert!(disposition.active_generation > previous_generation); + assert!( + engine + .fail_closed_reason() + .expect("quarantine reason") + .contains("candidate version 7 rejected") + ); + } + + #[test] + fn retain_validation_failure_keeps_previous_generation_active() { + let engine = OpaEngine::from_strings( + include_str!("../../openshell-supervisor-network/data/sandbox-policy.rego"), + "network_policies: {}\n", + ) + .unwrap(); + let previous_generation = engine.current_generation(); + + let quarantined = apply_policy_validation_failure( + &engine, + PolicyValidationFailureMode::FailClosed, + true, + 6, + "conflicting tls metadata", + ) + .unwrap(); + assert!(!quarantined.previous_policy_active); + + let disposition = apply_policy_validation_failure( + &engine, + PolicyValidationFailureMode::RetainLastValid, + true, + 7, + "conflicting tls metadata", + ) + .unwrap(); + + assert!(disposition.previous_policy_active); + assert!(disposition.active_generation > quarantined.active_generation); + assert!(disposition.active_generation > previous_generation); + assert!(engine.fail_closed_reason().is_none()); + } + + #[test] + fn retain_validation_failure_without_last_valid_policy_stays_fail_closed() { + let engine = OpaEngine::from_strings( + include_str!("../../openshell-supervisor-network/data/sandbox-policy.rego"), + "network_policies: {}\n", + ) + .unwrap(); + + let disposition = apply_policy_validation_failure( + &engine, + PolicyValidationFailureMode::RetainLastValid, + false, + 1, + "conflicting tls metadata", + ) + .unwrap(); + + assert_eq!( + disposition.configured_mode, + PolicyValidationFailureMode::RetainLastValid + ); + assert_eq!(disposition.mode, PolicyValidationFailureMode::FailClosed); + assert!(!disposition.previous_policy_active); + assert!(engine.fail_closed_reason().is_some()); + + let [config, _] = policy_validation_failure_events( + &disposition, + 1, + "sha256:test", + "conflicting tls metadata", + ); + let config = config.to_json().unwrap(); + assert_eq!(config["unmapped"]["validation_failure_mode"], "fail_closed"); + assert_eq!( + config["unmapped"]["configured_validation_failure_mode"], + "retain_last_valid" + ); + assert!( + config["message"] + .as_str() + .unwrap() + .contains("previous policy IS NOT active") + ); + } + + #[test] + fn validation_failure_ocsf_states_whether_previous_policy_is_active() { + let fail_closed = PolicyValidationFailureDisposition { + configured_mode: PolicyValidationFailureMode::FailClosed, + mode: PolicyValidationFailureMode::FailClosed, + previous_policy_active: false, + active_generation: 9, + }; + let [config, finding] = policy_validation_failure_events( + &fail_closed, + 8, + "sha256:test", + "conflicting tls metadata", + ); + let config = config.to_json().unwrap(); + assert_eq!(config["class_uid"], 5019); + assert_eq!(config["status"], "Failure"); + assert_eq!(config["unmapped"]["validation_failure_mode"], "fail_closed"); + assert_eq!( + config["unmapped"]["configured_validation_failure_mode"], + "fail_closed" + ); + assert_eq!(config["unmapped"]["previous_policy_active"], false); + assert!( + config["message"] + .as_str() + .unwrap() + .contains("previous policy IS NOT active") + ); + + let finding = finding.to_json().unwrap(); + assert_eq!(finding["class_uid"], 2004); + assert_eq!(finding["action"], "Denied"); + assert_eq!(finding["disposition"], "Blocked"); + + let retained = PolicyValidationFailureDisposition { + configured_mode: PolicyValidationFailureMode::RetainLastValid, + mode: PolicyValidationFailureMode::RetainLastValid, + previous_policy_active: true, + active_generation: 4, + }; + let [config, _] = policy_validation_failure_events( + &retained, + 8, + "sha256:test", + "conflicting tls metadata", + ); + let config = config.to_json().unwrap(); + assert_eq!(config["unmapped"]["previous_policy_active"], true); + assert!( + config["message"] + .as_str() + .unwrap() + .contains("previous policy IS active") + ); + } } diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index 1d942e4a3f..ee02393638 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -1731,6 +1731,9 @@ async fn handle_update_config_inner( "one of policy, setting_key, or merge_operations must be provided", )); } + if has_setting { + validate_setting_scope(key, req.global)?; + } if req.global { if !req.annotations.is_empty() { @@ -4323,6 +4326,16 @@ fn validate_registered_setting_key( }) } +fn validate_setting_scope(key: &str, global: bool) -> Result<(), Status> { + if key == settings::POLICY_VALIDATION_FAILURE_MODE_KEY && !global { + return Err(Status::invalid_argument(format!( + "setting '{}' is gateway-global and must be configured with --global", + settings::POLICY_VALIDATION_FAILURE_MODE_KEY + ))); + } + Ok(()) +} + fn proto_setting_to_stored(key: &str, value: &SettingValue) -> Result { let setting = validate_registered_setting_key(key)?; let expected = setting.kind; @@ -12221,6 +12234,15 @@ mod tests { assert!(err.message().contains("unknown setting key")); } + #[test] + fn policy_validation_failure_mode_is_gateway_global_only() { + assert!(validate_setting_scope(settings::POLICY_VALIDATION_FAILURE_MODE_KEY, true).is_ok()); + let error = validate_setting_scope(settings::POLICY_VALIDATION_FAILURE_MODE_KEY, false) + .expect_err("sandbox-scoped override must be rejected"); + assert_eq!(error.code(), Code::InvalidArgument); + assert!(error.message().contains("must be configured with --global")); + } + #[test] fn proto_setting_to_stored_rejects_policy_key() { let value = SettingValue { diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index 2d945b1a34..afc8b8e99c 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -20,6 +20,7 @@ use std::sync::{ Arc, Mutex, RwLock, atomic::{AtomicU64, Ordering}, }; +use tokio::sync::watch; use tracing::info; /// Baked-in rego rules for OPA policy evaluation. @@ -123,6 +124,8 @@ pub struct OpaEngine { engine: Mutex, generation: Arc, middleware_runner: RwLock, + generation_tx: watch::Sender, + fail_closed_reason: RwLock>, } #[cfg(test)] @@ -148,6 +151,7 @@ pub(crate) fn test_opa_query_count() -> u64 { pub struct PolicyGenerationGuard { captured_generation: u64, current_generation: Arc, + generation_rx: watch::Receiver, } impl PolicyGenerationGuard { @@ -173,6 +177,19 @@ impl PolicyGenerationGuard { } Ok(()) } + + /// Wait until the policy generation changes. + /// + /// Relay boundaries use this to close even an idle or raw stream as soon + /// as a new generation (including fail-closed quarantine) is published. + pub async fn wait_until_stale(&self) { + let mut receiver = self.generation_rx.clone(); + while !self.is_stale() { + if receiver.changed().await.is_err() { + return; + } + } + } } /// Per-tunnel L7 policy evaluator bound to the engine generation captured when @@ -219,6 +236,24 @@ impl TunnelPolicyEngine { } impl OpaEngine { + fn with_engine(engine: regorus::Engine) -> Self { + let generation = Arc::new(AtomicU64::new(0)); + let (generation_tx, _) = watch::channel(0); + Self { + engine: Mutex::new(engine), + generation, + middleware_runner: RwLock::new(ChainRunner::default()), + generation_tx, + fail_closed_reason: RwLock::new(None), + } + } + + fn advance_generation(&self) -> u64 { + let generation = self.generation.fetch_add(1, Ordering::AcqRel) + 1; + self.generation_tx.send_replace(generation); + generation + } + #[cfg(test)] pub(crate) fn poison_lock_for_test(&self) { let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { @@ -259,11 +294,7 @@ impl OpaEngine { engine .add_data_json(&data_json) .map_err(|e| miette::miette!("{e}"))?; - Ok(Self { - engine: Mutex::new(engine), - generation: Arc::new(AtomicU64::new(0)), - middleware_runner: RwLock::new(ChainRunner::default()), - }) + Ok(Self::with_engine(engine)) } /// Load policy rules and data from strings (data is YAML). @@ -314,11 +345,7 @@ impl OpaEngine { engine .add_data_json(&data_json) .map_err(|e| miette::miette!("{e}"))?; - Ok(Self { - engine: Mutex::new(engine), - generation: Arc::new(AtomicU64::new(0)), - middleware_runner: RwLock::new(ChainRunner::default()), - }) + Ok(Self::with_engine(engine)) } /// Create OPA engine from a typed proto policy. @@ -352,6 +379,18 @@ impl OpaEngine { entrypoint_pid: u32, require_binary_identity: bool, ) -> Result { + let ambiguities = openshell_policy::find_endpoint_ambiguities(proto); + if !ambiguities.is_empty() { + return Err(miette::miette!( + "network endpoint ambiguity validation failed:\n{}", + ambiguities + .iter() + .map(ToString::to_string) + .collect::>() + .join("\n") + )); + } + emit_binary_identity_mode(require_binary_identity, "proto"); if let Err(violations) = openshell_policy::validate_sandbox_policy(proto) { let errors = violations @@ -393,11 +432,7 @@ impl OpaEngine { engine .add_data_json(&data_json) .map_err(|e| miette::miette!("{e}"))?; - Ok(Self { - engine: Mutex::new(engine), - generation: Arc::new(AtomicU64::new(0)), - middleware_runner: RwLock::new(ChainRunner::default()), - }) + Ok(Self::with_engine(engine)) } /// Evaluate a network access request against the loaded policy. @@ -413,6 +448,19 @@ impl OpaEngine { .lock() .map_err(|_| miette::miette!("OPA engine lock poisoned"))?; + let fail_closed_reason = self + .fail_closed_reason + .read() + .map_err(|_| miette::miette!("OPA fail-closed state lock poisoned"))? + .clone(); + if let Some(reason) = fail_closed_reason { + return Ok(PolicyDecision { + allowed: false, + reason, + matched_policy: None, + }); + } + engine .set_input_json(&input_json.to_string()) .map_err(|e| miette::miette!("{e}"))?; @@ -467,6 +515,15 @@ impl OpaEngine { .map_err(|_| miette::miette!("OPA engine lock poisoned"))?; let generation = self.current_generation(); + let fail_closed_reason = self + .fail_closed_reason + .read() + .map_err(|_| miette::miette!("OPA fail-closed state lock poisoned"))? + .clone(); + if let Some(reason) = fail_closed_reason { + return Ok((NetworkAction::Deny { reason }, generation)); + } + engine .set_input_json(&input_json.to_string()) .map_err(|e| miette::miette!("{e}"))?; @@ -513,7 +570,11 @@ impl OpaEngine { .lock() .map_err(|_| miette::miette!("OPA engine lock poisoned"))?; *engine = new_engine; - self.generation.fetch_add(1, Ordering::AcqRel); + *self + .fail_closed_reason + .write() + .map_err(|_| miette::miette!("OPA fail-closed state lock poisoned"))? = None; + self.advance_generation(); Ok(()) } @@ -548,7 +609,11 @@ impl OpaEngine { .lock() .map_err(|_| miette::miette!("OPA engine lock poisoned"))?; *engine = new_engine; - self.generation.fetch_add(1, Ordering::AcqRel); + *self + .fail_closed_reason + .write() + .map_err(|_| miette::miette!("OPA fail-closed state lock poisoned"))? = None; + self.advance_generation(); Ok(()) } @@ -583,10 +648,61 @@ impl OpaEngine { .map_err(|_| miette::miette!("middleware runner lock poisoned"))?; *engine = new_engine; *runner = new_runner; - self.generation.fetch_add(1, Ordering::AcqRel); + *self + .fail_closed_reason + .write() + .map_err(|_| miette::miette!("OPA fail-closed state lock poisoned"))? = None; + self.advance_generation(); Ok(()) } + /// Publish a deny-all quarantine generation without activating any part + /// of the invalid candidate policy. + /// + /// The existing compiled engine remains available for an explicit + /// `retain_last_valid` posture or a later valid reload, but all new network + /// decisions deny with `reason` while the quarantine is active. Advancing + /// the generation invalidates and wakes every pinned relay. + pub fn enter_fail_closed(&self, reason: impl Into) -> Result { + let _engine = self + .engine + .lock() + .map_err(|_| miette::miette!("OPA engine lock poisoned"))?; + *self + .fail_closed_reason + .write() + .map_err(|_| miette::miette!("OPA fail-closed state lock poisoned"))? = + Some(reason.into()); + Ok(self.advance_generation()) + } + + pub fn fail_closed_reason(&self) -> Option { + self.fail_closed_reason + .read() + .ok() + .and_then(|reason| reason.clone()) + } + + /// Reactivate the compiled last-known-good engine after an operator + /// explicitly selects the availability-oriented retention posture. + pub fn exit_fail_closed(&self) -> Result { + let _engine = self + .engine + .lock() + .map_err(|_| miette::miette!("OPA engine lock poisoned"))?; + let was_fail_closed = self + .fail_closed_reason + .write() + .map_err(|_| miette::miette!("OPA fail-closed state lock poisoned"))? + .take() + .is_some(); + if was_fail_closed { + Ok(self.advance_generation()) + } else { + Ok(self.current_generation()) + } + } + /// Current policy generation. Successful reloads increment this value. pub fn current_generation(&self) -> u64 { self.generation.load(Ordering::Acquire) @@ -600,7 +716,7 @@ impl OpaEngine { .write() .map_err(|_| miette::miette!("middleware runner lock poisoned"))?; *runner = ChainRunner::from_registry(registry); - self.generation.fetch_add(1, Ordering::AcqRel); + self.advance_generation(); Ok(()) } @@ -633,6 +749,7 @@ impl OpaEngine { Ok(PolicyGenerationGuard { captured_generation: generation, current_generation: Arc::clone(&self.generation), + generation_rx: self.generation_tx.subscribe(), }) } @@ -798,6 +915,7 @@ impl OpaEngine { generation_guard: PolicyGenerationGuard { captured_generation: generation, current_generation: Arc::clone(&self.generation), + generation_rx: self.generation_tx.subscribe(), }, middleware_runner: self.middleware_runner()?, }) @@ -3079,11 +3197,7 @@ network_policies: .expect("policy should load"); rego.add_data_json(&data_json.to_string()) .expect("data should load"); - let engine = OpaEngine { - engine: Mutex::new(rego), - generation: Arc::new(AtomicU64::new(0)), - middleware_runner: RwLock::new(ChainRunner::default()), - }; + let engine = OpaEngine::with_engine(rego); let input = l7_websocket_graphql_input( "realtime.graphql.com", serde_json::json!([{ @@ -4712,6 +4826,97 @@ network_policies: assert_eq!(val, regorus::Value::from(true)); } + #[test] + fn proto_load_rejects_ambiguous_endpoint_metadata_with_rationale() { + let mut policy = ProtoSandboxPolicy::default(); + policy.network_policies.insert( + "wildcard".to_string(), + NetworkPolicyRule { + name: "wildcard".to_string(), + endpoints: vec![NetworkEndpoint { + host: "*.example.com".to_string(), + port: 443, + tls: "skip".to_string(), + ..Default::default() + }], + binaries: vec![NetworkBinary { + path: "/usr/bin/curl".to_string(), + ..Default::default() + }], + }, + ); + policy.network_policies.insert( + "exact".to_string(), + NetworkPolicyRule { + name: "exact".to_string(), + endpoints: vec![NetworkEndpoint { + host: "api.example.com".to_string(), + port: 443, + ..Default::default() + }], + binaries: vec![NetworkBinary { + path: "/usr/bin/bash".to_string(), + ..Default::default() + }], + }, + ); + + let Err(error) = OpaEngine::from_proto(&policy) else { + panic!("ambiguity must reject activation"); + }; + let message = error.to_string(); + assert!(message.contains("ambiguity validation failed")); + assert!(message.contains("wildcard")); + assert!(message.contains("exact")); + assert!(message.contains("tls")); + } + + #[tokio::test] + async fn fail_closed_quarantine_denies_and_wakes_generation_guards() { + let engine = test_engine(); + let guard = engine + .generation_guard(engine.current_generation()) + .unwrap(); + let stale = guard.wait_until_stale(); + + let generation = engine + .enter_fail_closed("candidate policy validation failed: conflicting tls") + .unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(1), stale) + .await + .expect("generation waiter should wake"); + assert_eq!(generation, 1); + assert!(guard.is_stale()); + + let input = NetworkInput { + host: "api.anthropic.com".to_string(), + port: 443, + binary_path: PathBuf::from("/usr/bin/curl"), + binary_sha256: String::new(), + ancestors: vec![], + cmdline_paths: vec![], + }; + let action = engine.evaluate_network_action(&input).unwrap(); + assert_eq!( + action, + NetworkAction::Deny { + reason: "candidate policy validation failed: conflicting tls".to_string() + } + ); + } + + #[test] + fn valid_reload_exits_fail_closed_quarantine() { + let engine = test_engine(); + engine.enter_fail_closed("invalid candidate").unwrap(); + assert!(engine.fail_closed_reason().is_some()); + + engine.reload(TEST_POLICY, TEST_DATA_YAML).unwrap(); + + assert!(engine.fail_closed_reason().is_none()); + assert_eq!(engine.current_generation(), 2); + } + #[test] fn endpoint_config_generation_matches_query_generation() { let engine = l7_engine(); From 66bfc03af8c427448ed876a5c618c66cabda69a9 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Mon, 20 Jul 2026 09:50:35 -0700 Subject: [PATCH 09/30] refactor(network): invalidate relays on policy changes Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- .../openshell-supervisor-network/src/proxy.rs | 140 +++++++++++------ .../src/proxy/relay.rs | 147 ++++++++++++++---- .../src/proxy/tests/phase0.rs | 9 +- 3 files changed, 216 insertions(+), 80 deletions(-) diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index c2617d499a..6452223eb7 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -869,6 +869,38 @@ fn build_forward_allow_ocsf_event( .build() } +#[allow(clippy::too_many_arguments)] +fn build_forward_policy_deny_ocsf_event( + peer_addr: SocketAddr, + method: &str, + host: &str, + port: u16, + path: &str, + binary: &str, + pid: &str, + ancestors: &str, + cmdline: &str, + reason: &str, +) -> openshell_ocsf::OcsfEvent { + HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Other) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .http_request(HttpRequest::new( + method, + OcsfUrl::new("http", host, path, port), + )) + .dst_endpoint(Endpoint::from_domain(host, port)) + .src_endpoint(Endpoint::from_ip(peer_addr.ip(), peer_addr.port())) + .actor_process(Process::from_bypass(binary, pid, ancestors).with_cmd_line(cmdline)) + .firewall_rule("-", "opa") + .message(format!("FORWARD denied {method} {host}:{port}{path}")) + .status_detail(reason) + .build() +} + #[allow(clippy::too_many_arguments)] async fn deny_connect_destination( client: &mut TcpStream, @@ -1433,7 +1465,11 @@ async fn handle_tcp_connection( port = port, "tls: skip — bypassing TLS auto-detection, raw tunnel" ); - relay::relay_tcp(&mut client, &mut upstream).await?; + let Some(generation_guard) = relay::prepare_raw_relay(l7_route, &opa_engine, &decision) + else { + return Ok(()); + }; + relay::relay_tcp(&mut client, &mut upstream, &generation_guard, &ctx).await?; return Ok(()); } @@ -1614,7 +1650,11 @@ async fn handle_tcp_connection( port = port, "Non-TLS non-HTTP traffic detected, raw tunnel" ); - relay::relay_tcp(&mut client, &mut upstream).await?; + let Some(generation_guard) = relay::prepare_raw_relay(l7_route, &opa_engine, &decision) + else { + return Ok(()); + }; + relay::relay_tcp(&mut client, &mut upstream, &generation_guard, &ctx).await?; } Ok(()) @@ -3923,28 +3963,18 @@ async fn handle_forward_proxy( let matched_policy = match &decision.action { NetworkAction::Allow { matched_policy } => matched_policy.clone(), NetworkAction::Deny { reason } => { - { - let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Other) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .http_request(HttpRequest::new( - method, - OcsfUrl::new("http", &host_lc, &path, port), - )) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .src_endpoint(Endpoint::from_ip(workload_addr.ip(), workload_addr.port())) - .actor_process( - Process::from_bypass(&binary_str, &pid_str, &ancestors_str) - .with_cmd_line(&cmdline_str), - ) - .firewall_rule("-", "opa") - .message(format!("FORWARD denied {method} {host_lc}:{port}{path}")) - .build(); - ocsf_emit!(event); - } + ocsf_emit!(build_forward_policy_deny_ocsf_event( + workload_addr, + method, + &host_lc, + port, + &path, + &binary_str, + &pid_str, + &ancestors_str, + &cmdline_str, + reason, + )); emit_denial_simple( denial_tx, &host_lc, @@ -4034,8 +4064,10 @@ async fn handle_forward_proxy( let mut l7_activity_pending = false; // 4b. If the endpoint has L7 config, evaluate the request against - // L7 policy. The forward proxy handles exactly one request per - // connection (Connection: close), so a single evaluation suffices. + // L7 policy. The forward proxy handles exactly one request per + // connection, so a single evaluation suffices. The shared HTTP relay + // strips hop-by-hop `Connection` headers and drops the upstream after + // the response instead of asking the upstream to close it. hydrate_l7_route(&opa_engine, &mut decision); if let Some(route) = decision .endpoint @@ -4774,28 +4806,18 @@ async fn handle_forward_proxy( // The request has now survived middleware, token grant, credential // rewriting, generation checks, and the HTTP relay. Only now record the // final allowed outcome. - { - let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Other) - .action(ActionId::Allowed) - .disposition(DispositionId::Allowed) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .http_request(HttpRequest::new( - method, - OcsfUrl::new("http", &host_lc, &path, port), - )) - .dst_endpoint(Endpoint::from_domain(&host_lc, port)) - .src_endpoint(Endpoint::from_ip(workload_addr.ip(), workload_addr.port())) - .actor_process( - Process::from_bypass(&binary_str, &pid_str, &ancestors_str) - .with_cmd_line(&cmdline_str), - ) - .firewall_rule(policy_str, "opa") - .message(format!("FORWARD allowed {method} {host_lc}:{port}{path}")) - .build(); - ocsf_emit!(event); - } + ocsf_emit!(build_forward_allow_ocsf_event( + workload_addr, + method, + &host_lc, + port, + &path, + &binary_str, + &pid_str, + &ancestors_str, + &cmdline_str, + policy_str, + )); emit_forward_success_activity(activity_tx, l7_activity_pending); if let crate::l7::provider::RelayOutcome::Upgraded { @@ -5213,6 +5235,28 @@ network_policies: {} assert!(body.get("reason").is_none()); } + #[test] + fn forward_policy_denial_ocsf_includes_validation_rationale() { + let reason = "policy validation failed; fail-closed quarantine is active; candidate version 7 rejected: conflicting tls metadata"; + let event = build_forward_policy_deny_ocsf_event( + "127.0.0.1:45123".parse().unwrap(), + "GET", + "api.example.com", + 80, + "/v1/models", + "/usr/bin/curl", + "42", + "/usr/bin/bash", + "curl http://api.example.com/v1/models", + reason, + ); + let json = event.to_json().unwrap(); + + assert_eq!(json["status_detail"], reason); + assert_eq!(json["action"], "Denied"); + assert_eq!(json["disposition"], "Blocked"); + } + #[test] fn endpoint_only_opa_allows_declared_endpoint_without_process_identity() { let policy = include_str!("../data/sandbox-policy.rego"); diff --git a/crates/openshell-supervisor-network/src/proxy/relay.rs b/crates/openshell-supervisor-network/src/proxy/relay.rs index b025b5ceb4..cdc49f95bc 100644 --- a/crates/openshell-supervisor-network/src/proxy/relay.rs +++ b/crates/openshell-supervisor-network/src/proxy/relay.rs @@ -102,7 +102,7 @@ pub(super) fn pin_l7_evaluator( /// attempting to write an HTTP response into it. pub(super) fn prepare_http_relay<'a>( route: Option<&L7RouteSnapshot>, - opa_engine: &OpaEngine, + opa_engine: &'a OpaEngine, decision: &EgressDecision, request: &'a L7EvalContext, ) -> Option> { @@ -152,6 +152,30 @@ pub(super) fn prepare_http_relay<'a>( }) } +/// Pin the generation used by a raw relay so policy activation or quarantine +/// closes streams that otherwise have no request boundary at which to notice +/// a stale decision. +pub(super) fn prepare_raw_relay( + route: Option<&L7RouteSnapshot>, + opa_engine: &OpaEngine, + decision: &EgressDecision, +) -> Option { + let expected_generation = route.map_or(decision.l4_policy_generation, |route| { + route.l7_policy_generation + }); + match pin_policy_generation(opa_engine, expected_generation) { + Ok(guard) => Some(guard), + Err(error) => { + emit_l7_tunnel_close_after_policy_change( + &decision.intent.destination.host, + decision.intent.destination.port, + error, + ); + None + } + } +} + /// Relay an HTTP/1 stream using an already-authorized, generation-pinned context. /// /// CONNECT plaintext and TLS-terminated streams both enter through this @@ -168,50 +192,89 @@ where { match context.policy { PreparedHttpPolicy::Inspect { configs, evaluator } if configs.len() == 1 => { - crate::l7::relay::relay_with_inspection( - &configs[0], - *evaluator, - client, - upstream, - context.request, - ) - .await + let generation_guard = evaluator.generation_guard().clone(); + tokio::select! { + result = crate::l7::relay::relay_with_inspection( + &configs[0], + *evaluator, + client, + upstream, + context.request, + ) => result, + () = generation_guard.wait_until_stale() => { + emit_stale_relay_close(context.request, &generation_guard); + Ok(()) + } + } } PreparedHttpPolicy::Inspect { configs, evaluator } => { - crate::l7::relay::relay_with_route_selection( - &configs, - *evaluator, - client, - upstream, - context.request, - ) - .await + let generation_guard = evaluator.generation_guard().clone(); + tokio::select! { + result = crate::l7::relay::relay_with_route_selection( + &configs, + *evaluator, + client, + upstream, + context.request, + ) => result, + () = generation_guard.wait_until_stale() => { + emit_stale_relay_close(context.request, &generation_guard); + Ok(()) + } + } } PreparedHttpPolicy::Passthrough { generation_guard } => { - crate::l7::relay::relay_passthrough_with_credentials( - client, - upstream, - context.request, - &generation_guard, - Some(context.middleware_engine), - ) - .await + tokio::select! { + result = crate::l7::relay::relay_passthrough_with_credentials( + client, + upstream, + context.request, + &generation_guard, + Some(context.middleware_engine), + ) => result, + () = generation_guard.wait_until_stale() => { + emit_stale_relay_close(context.request, &generation_guard); + Ok(()) + } + } } } } /// Relay a policy-authorized raw TCP stream. -pub(super) async fn relay_tcp(client: &mut C, upstream: &mut U) -> Result<()> +pub(super) async fn relay_tcp( + client: &mut C, + upstream: &mut U, + generation_guard: &PolicyGenerationGuard, + request: &L7EvalContext, +) -> Result<()> where C: AsyncRead + AsyncWrite + Unpin, U: AsyncRead + AsyncWrite + Unpin, { - tokio::io::copy_bidirectional(client, upstream) - .await - .into_diagnostic()?; + tokio::select! { + result = tokio::io::copy_bidirectional(client, upstream) => { + result.into_diagnostic()?; + } + () = generation_guard.wait_until_stale() => { + emit_stale_relay_close(request, generation_guard); + } + } Ok(()) } +fn emit_stale_relay_close(request: &L7EvalContext, guard: &PolicyGenerationGuard) { + emit_l7_tunnel_close_after_policy_change( + &request.host, + request.port, + miette::miette!( + "policy generation is stale [captured_generation:{} current_generation:{}]", + guard.captured_generation(), + guard.current_generation(), + ), + ); +} + #[cfg(test)] mod tests { use super::super::{EgressIntent, EndpointDecision, ProcessIdentityEvidence}; @@ -303,4 +366,30 @@ mod tests { "policy reload must prevent a stale relay from starting" ); } + + #[tokio::test] + async fn raw_relay_closes_immediately_when_fail_closed_generation_is_published() { + let engine = Arc::new(OpaEngine::from_strings(POLICY_REGO, EMPTY_POLICY_DATA).unwrap()); + let guard = engine + .generation_guard(engine.current_generation()) + .unwrap(); + let request = request_context(); + let (_client_peer, mut proxy_client) = tokio::io::duplex(64); + let (_upstream_peer, mut proxy_upstream) = tokio::io::duplex(64); + + let relay = tokio::spawn(async move { + relay_tcp(&mut proxy_client, &mut proxy_upstream, &guard, &request).await + }); + tokio::task::yield_now().await; + + engine + .enter_fail_closed("candidate policy validation failed") + .unwrap(); + + tokio::time::timeout(std::time::Duration::from_secs(1), relay) + .await + .expect("raw relay should close when its generation becomes stale") + .expect("relay task should not panic") + .expect("stale relay closure should be clean"); + } } diff --git a/crates/openshell-supervisor-network/src/proxy/tests/phase0.rs b/crates/openshell-supervisor-network/src/proxy/tests/phase0.rs index a3f3260b9f..527c0b2dc1 100644 --- a/crates/openshell-supervisor-network/src/proxy/tests/phase0.rs +++ b/crates/openshell-supervisor-network/src/proxy/tests/phase0.rs @@ -461,7 +461,8 @@ fn forward_rewrite_does_not_treat_a_pipelined_request_as_body_overflow() { POST http://target.example/blocked HTTP/1.1\r\n\ Host: target.example\r\n\ Content-Length: 0\r\n\r\n"; - let rewritten = rewrite_forward_request(raw, raw.len(), "/allowed", None, false).unwrap(); + let rewritten = + rewrite_forward_request(raw, raw.len(), "/allowed", "target.example", None, false).unwrap(); let rewritten = String::from_utf8(rewritten).unwrap(); assert!(rewritten.starts_with("GET /allowed HTTP/1.1\r\n")); @@ -477,7 +478,8 @@ fn forward_rewrite_trims_pipeline_after_content_length_body() { body\ GET http://target.example/blocked HTTP/1.1\r\n\ Host: target.example\r\n\r\n"; - let rewritten = rewrite_forward_request(raw, raw.len(), "/allowed", None, false).unwrap(); + let rewritten = + rewrite_forward_request(raw, raw.len(), "/allowed", "target.example", None, false).unwrap(); let rewritten = String::from_utf8(rewritten).unwrap(); assert!(rewritten.ends_with("\r\n\r\nbody")); @@ -492,7 +494,8 @@ fn forward_rewrite_trims_pipeline_after_complete_chunked_body() { 4\r\nbody\r\n0\r\n\r\n\ GET http://target.example/blocked HTTP/1.1\r\n\ Host: target.example\r\n\r\n"; - let rewritten = rewrite_forward_request(raw, raw.len(), "/allowed", None, false).unwrap(); + let rewritten = + rewrite_forward_request(raw, raw.len(), "/allowed", "target.example", None, false).unwrap(); let rewritten = String::from_utf8(rewritten).unwrap(); assert!(rewritten.ends_with("4\r\nbody\r\n0\r\n\r\n")); From 1a78bb9e552a10565636a457fae70a8899e00b01 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Mon, 20 Jul 2026 09:50:38 -0700 Subject: [PATCH 10/30] docs(policy): document validation failure posture Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- architecture/security-policy.md | 27 ++++++++++++++++++++------- docs/sandboxes/policies.mdx | 29 +++++++++++++++++++++++++++-- 2 files changed, 47 insertions(+), 9 deletions(-) diff --git a/architecture/security-policy.md b/architecture/security-policy.md index b4f0bdb912..5e18b0d176 100644 --- a/architecture/security-policy.md +++ b/architecture/security-policy.md @@ -82,9 +82,9 @@ metadata before forwarding. The proxy also supports credential injection on terminated HTTP streams when policy allows the endpoint. Raw streams and long-lived response bodies are connection scoped. Policy -reloads affect the next connection or the next parsed HTTP request; they do not -rewrite bytes already being relayed. HTTP upgrades switch to raw relay by -default. A `protocol: rest` endpoint can opt in to +generation changes close relays pinned to the previous generation instead of +allowing them to continue under stale authorization. HTTP upgrades switch to +raw relay by default. A `protocol: rest` endpoint can opt in to `websocket_credential_rewrite` for client-to-server WebSocket text messages after an allowed `101` upgrade; server-to-client traffic and all other upgraded protocols remain raw passthrough. @@ -98,10 +98,23 @@ supervisor polls for config revisions and attempts to load new dynamic policy into the in-process OPA engine; CLI reads of the latest sandbox policy use the same effective configuration path. -If a new policy fails validation or loading, the supervisor reports the failure -and keeps the last-known-good policy. Static controls, such as filesystem -allowlists and process identity, require a new sandbox because they are applied -before the child process starts. +The supervisor validates complete effective policy generations before +activation. Overlapping endpoint selectors may contribute request allow and +deny rules only when their connection and request-processing metadata agree; +conflicting TLS, destination, credential, parser, or enforcement metadata +rejects the complete generation. + +The gateway-global `policy_validation_failure_mode` setting controls rejected +generations. It defaults to `fail_closed`, which publishes a quarantine +generation, denies new egress, invalidates existing relays, and leaves the +previous policy inactive. Operators may explicitly select +`retain_last_valid`, which keeps the previous generation active. With no +previous valid generation, the effective mode remains `fail_closed` regardless +of the configured mode. OCSF configuration and finding events state the +candidate version, validation rationale, configured and effective modes, active +generation, and whether the previous policy is active. Static controls, +such as filesystem allowlists and process identity, require a new sandbox +because they are applied before the child process starts. Gateway-global policy can override sandbox-scoped policy. Use it sparingly because it changes the effective access model for every sandbox on the gateway. diff --git a/docs/sandboxes/policies.mdx b/docs/sandboxes/policies.mdx index 4c2e12dacf..2f6150c962 100644 --- a/docs/sandboxes/policies.mdx +++ b/docs/sandboxes/policies.mdx @@ -61,8 +61,7 @@ network_middlewares: Static sections are locked at sandbox creation. Changing them requires destroying and recreating the sandbox. Dynamic sections can be updated on a running sandbox with `openshell policy update` for incremental merges or `openshell policy set` for full replacement, and take effect without restarting. -When a hot reload changes rules on an active HTTP L7 endpoint, existing keep-alive tunnels are closed before forwarding another parsed request. Credential-injection-only HTTP passthrough tunnels use the same reload boundary. Most HTTP clients reconnect automatically, and the next request is evaluated against the current policy. -Raw streams are connection-scoped and outside L7 live-reload guarantees. This includes `tls: skip`, non-HTTP TCP payloads, HTTP upgrades such as WebSocket, and long-lived response streams such as SSE. A reload applies to the next connection or next parsed HTTP request; it does not interrupt an already-forwarded raw stream. Use `protocol: websocket` when policy should stay attached to the RFC 6455 upgrade and client text messages after the allowed upgrade. Add `websocket_credential_rewrite: true` only when the relay should rewrite credential placeholders in client-to-server WebSocket text messages. Add `request_body_credential_rewrite: true` only on inspected REST endpoints that need OpenShell to rewrite placeholders in supported text request bodies. +When a hot reload changes rules, the supervisor publishes a new policy generation and closes connections pinned to the previous generation. This includes HTTP keep-alive tunnels, `tls: skip`, non-HTTP payloads, HTTP upgrades such as WebSocket, and long-lived response streams such as SSE. Most clients reconnect automatically, and the next connection or request is evaluated against the current policy. Use `protocol: websocket` when policy should stay attached to the RFC 6455 upgrade and client text messages after the allowed upgrade. Add `websocket_credential_rewrite: true` only when the relay should rewrite credential placeholders in client-to-server WebSocket text messages. Add `request_body_credential_rewrite: true` only on inspected REST endpoints that need OpenShell to rewrite placeholders in supported text request bodies. | Section | Type | Description | |---|---|---| @@ -202,6 +201,32 @@ The following steps outline the hot-reload policy update workflow. openshell policy list ``` +### Validation failures + +OpenShell validates a complete candidate policy before activating any part of it. Endpoints may overlap when their connection and request-processing metadata agree. For example, two `api.example.com:443` REST entries can contribute different allow and deny rules when they use the same TLS, destination, credential, parser, and enforcement settings. OpenShell rejects the candidate when overlapping exact or wildcard host selectors disagree on those fields. Path-scoped endpoints may use different protocols only when their path selectors cannot match the same request. + +The gateway-global `policy_validation_failure_mode` setting determines what happens after rejection. Its default is `fail_closed`: + +```shell +openshell settings set --global \ + --key policy_validation_failure_mode \ + --value fail_closed +``` + +In `fail_closed` mode, the supervisor publishes a quarantine generation, denies new egress, and closes connections pinned to the previous generation. The previous policy is not active. A later valid policy exits quarantine automatically. + +Operators that explicitly prioritize availability can retain the previous generation: + +```shell +openshell settings set --global \ + --key policy_validation_failure_mode \ + --value retain_last_valid +``` + +In `retain_last_valid` mode, the rejected candidate remains inactive and the previous valid generation remains active. If no previous valid generation exists, such as during initial startup, OpenShell still fails closed. The setting is gateway-global and cannot be overridden by an individual sandbox. + +OCSF configuration and finding events identify the rejected candidate, validation rationale, configured and effective modes, active generation, and whether the previous policy is active. When `retain_last_valid` is configured without a previous valid generation, the effective mode remains `fail_closed`. Connection denials during quarantine include the validation failure as their policy denial rationale. + ## Incremental Policy Updates Use `openshell policy update` when you want to merge network policy changes into the current live policy instead of replacing the whole YAML document. This command only updates the dynamic `network_policies` section. From 9dd94a123bf4dfc0f1442c876e38ee04405df007 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Mon, 20 Jul 2026 09:50:40 -0700 Subject: [PATCH 11/30] test(network): cover validation and middleware egress Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- e2e/python/test_sandbox_policy.py | 16 +- e2e/rust/tests/proxy_egress_pipeline.rs | 599 ++++++++++++++++++++++-- 2 files changed, 570 insertions(+), 45 deletions(-) diff --git a/e2e/python/test_sandbox_policy.py b/e2e/python/test_sandbox_policy.py index 5ac37bd27f..82e32a9b34 100644 --- a/e2e/python/test_sandbox_policy.py +++ b/e2e/python/test_sandbox_policy.py @@ -1950,15 +1950,14 @@ def test_host_wildcard_rejects_deep_subdomain( # ============================================================================= -def test_overlapping_policies_do_not_crash_opa( +def test_overlapping_policies_with_conflicting_destination_metadata_are_rejected( sandbox: Callable[..., Sandbox], ) -> None: - """OVL-1: Two policies covering the same host:port must not crash OPA. + """OVL-1: Conflicting metadata on the same host:port fails closed. - After a draft rule approval, the merged policy can contain two entries - for the same (host, port). The OPA engine must handle this without - a 'duplicated definition of local variable' error. This test creates - the overlap directly to simulate the post-approval state. + One endpoint permits any resolved address while the other constrains + ``allowed_ips``. The complete candidate is ambiguous and must not activate + either entry. """ policy = _base_policy( network_policies={ @@ -1992,8 +1991,9 @@ def test_overlapping_policies_do_not_crash_opa( args=(_PROXY_HOST, _PROXY_PORT, _SANDBOX_IP, _FORWARD_PROXY_PORT), ) assert result.exit_code == 0, result.stderr - assert "200" in result.stdout, ( - f"Overlapping policies should not crash; expected 200, got: {result.stdout}" + assert "403" in result.stdout, ( + "Conflicting overlapping policies should fail closed; " + f"expected 403, got: {result.stdout}" ) diff --git a/e2e/rust/tests/proxy_egress_pipeline.rs b/e2e/rust/tests/proxy_egress_pipeline.rs index 88b78ac181..28d2d6f1d5 100644 --- a/e2e/rust/tests/proxy_egress_pipeline.rs +++ b/e2e/rust/tests/proxy_egress_pipeline.rs @@ -16,7 +16,10 @@ use std::io::{self, Error, ErrorKind, Write}; use std::process::Stdio; -use std::sync::{Arc, Mutex}; +use std::sync::{ + Arc, Mutex, + atomic::{AtomicUsize, Ordering}, +}; use openshell_e2e::harness::binary::openshell_cmd; use openshell_e2e::harness::sandbox::SandboxGuard; @@ -84,7 +87,12 @@ async fn create_generic_provider(name: &str) -> Result { .await } -fn write_policy(host: &str, port: u16, endpoint_options: &str) -> Result { +fn write_policy_document( + host: &str, + port: u16, + endpoint_options: &str, + network_middlewares: &str, +) -> Result { let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; let policy = format!( r#"version: 1 @@ -111,6 +119,7 @@ process: run_as_user: sandbox run_as_group: sandbox +{network_middlewares} network_policies: proxy_egress_test: name: proxy_egress_test @@ -130,6 +139,33 @@ network_policies: Ok(file) } +fn write_policy(host: &str, port: u16, endpoint_options: &str) -> Result { + write_policy_document(host, port, endpoint_options, "") +} + +fn write_middleware_policy( + host: &str, + port: u16, + endpoint_options: &str, + on_error: &str, +) -> Result { + let network_middlewares = format!( + r#"network_middlewares: + regex-redactor: + name: Redact API tokens + middleware: openshell/regex + order: 10 + config: + mode: redact + on_error: {on_error} + endpoints: + include: ["{host}"] + exclude: [] +"# + ); + write_policy_document(host, port, endpoint_options, &network_middlewares) +} + fn write_denied_policy() -> Result { let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; let policy = r#"version: 1 @@ -165,6 +201,50 @@ network_policies: {} Ok(file) } +fn write_ambiguous_policy(host: &str, port: u16) -> Result { + let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; + let policy = format!( + r#"version: 1 + +filesystem_policy: + include_workdir: true + read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] + read_write: [/sandbox, /tmp, /dev/null] + +landlock: + compatibility: best_effort + +process: + run_as_user: sandbox + run_as_group: sandbox + +network_policies: + terminating: + name: terminating + endpoints: + - host: {host} + port: {port} +{PRIVATE_ALLOWED_IPS} + binaries: + - path: "/**" + passthrough: + name: passthrough + endpoints: + - host: {host} + port: {port} + tls: skip +{PRIVATE_ALLOWED_IPS} + binaries: + - path: "/**" +"# + ); + file.write_all(policy.as_bytes()) + .map_err(|error| format!("write policy: {error}"))?; + file.flush() + .map_err(|error| format!("flush policy: {error}"))?; + Ok(file) +} + fn write_destination_denial_policy() -> Result { let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; let policy = r#"version: 1 @@ -309,6 +389,7 @@ async fn read_http_request(stream: &mut TcpStream) -> io::Result> struct KeepAliveHttpServer { port: u16, + connections: Arc, task: JoinHandle<()>, } @@ -321,14 +402,25 @@ impl KeepAliveHttpServer { .local_addr() .map_err(|error| format!("read HTTP server address: {error}"))? .port(); + let connections = Arc::new(AtomicUsize::new(0)); + let task_connections = Arc::clone(&connections); let task = tokio::spawn(async move { while let Ok((stream, _)) = listener.accept().await { + task_connections.fetch_add(1, Ordering::AcqRel); tokio::spawn(async move { let _ = handle_keep_alive_connection(stream).await; }); } }); - Ok(Self { port, task }) + Ok(Self { + port, + connections, + task, + }) + } + + fn connection_count(&self) -> usize { + self.connections.load(Ordering::Acquire) } } @@ -356,9 +448,56 @@ async fn handle_keep_alive_connection(mut stream: TcpStream) -> io::Result<()> { struct EchoServer { port: u16, + observed: Arc>>, task: JoinHandle<()>, } +struct RequestBodyEchoServer { + port: u16, + task: JoinHandle<()>, +} + +impl RequestBodyEchoServer { + async fn start() -> Result { + let listener = TcpListener::bind(("0.0.0.0", 0)) + .await + .map_err(|error| format!("bind request body echo server: {error}"))?; + let port = listener + .local_addr() + .map_err(|error| format!("read request body echo server address: {error}"))? + .port(); + let task = tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + tokio::spawn(async move { + let _ = handle_request_body_echo(stream).await; + }); + } + }); + Ok(Self { port, task }) + } +} + +impl Drop for RequestBodyEchoServer { + fn drop(&mut self) { + self.task.abort(); + } +} + +async fn handle_request_body_echo(mut stream: TcpStream) -> io::Result<()> { + let request = read_http_request(&mut stream) + .await? + .ok_or_else(|| Error::new(ErrorKind::UnexpectedEof, "missing HTTP request"))?; + let headers_end = header_end(&request) + .ok_or_else(|| Error::new(ErrorKind::UnexpectedEof, "incomplete HTTP headers"))?; + let body = &request[headers_end..]; + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + stream.write_all(response.as_bytes()).await?; + stream.write_all(body).await +} + struct PipelineProbeServer { port: u16, observed: Arc>>, @@ -430,22 +569,37 @@ impl EchoServer { .local_addr() .map_err(|error| format!("read echo server address: {error}"))? .port(); + let observed = Arc::new(Mutex::new(Vec::new())); + let task_observed = Arc::clone(&observed); let task = tokio::spawn(async move { while let Ok((mut stream, _)) = listener.accept().await { + let observed = Arc::clone(&task_observed); tokio::spawn(async move { let mut buffer = [0_u8; 4096]; loop { let Ok(read) = stream.read(&mut buffer).await else { break; }; - if read == 0 || stream.write_all(&buffer[..read]).await.is_err() { + if read == 0 { + break; + } + observed.lock().unwrap().extend_from_slice(&buffer[..read]); + if stream.write_all(&buffer[..read]).await.is_err() { break; } } }); } }); - Ok(Self { port, task }) + Ok(Self { + port, + observed, + task, + }) + } + + fn observed_bytes(&self) -> Vec { + self.observed.lock().unwrap().clone() } } @@ -789,6 +943,140 @@ async fn policy_reload_updates_both_adapters_and_closes_existing_http_tunnel() { guard.cleanup().await; } +#[tokio::test] +#[allow(clippy::too_many_lines)] +async fn ambiguous_policy_update_fails_closed_without_contacting_upstream() { + let server = KeepAliveHttpServer::start() + .await + .expect("start keep-alive HTTP server"); + let valid_policy = write_policy(TEST_SERVER_HOST, server.port, "").expect("write valid policy"); + let ambiguous_policy = + write_ambiguous_policy(TEST_SERVER_HOST, server.port).expect("write ambiguous policy"); + let valid_policy_path = policy_path(&valid_policy); + let ambiguous_policy_path = policy_path(&ambiguous_policy); + + let mut guard = SandboxGuard::create_keep_with_args( + &["--policy", &valid_policy_path], + &["sh", "-c", "echo Ready; sleep infinity"], + "Ready", + ) + .await + .expect("create keep sandbox"); + + run_cli(&[ + "policy", + "set", + &guard.name, + "--policy", + &valid_policy_path, + "--wait", + "--timeout", + "120", + ]) + .await + .expect("wait for valid policy"); + + let status_script = proxy_status_script(TEST_SERVER_HOST, server.port); + let before = guard + .exec(&["python3", "-c", &status_script]) + .await + .expect("exercise both adapters before invalid update"); + let before = parse_json_line(&before); + assert_eq!(before["connect"], 200, "CONNECT before update: {before}"); + assert_eq!(before["forward"], 200, "forward before update: {before}"); + let connections_before_rejection = server.connection_count(); + + let update_error = run_cli(&[ + "policy", + "set", + &guard.name, + "--policy", + &ambiguous_policy_path, + "--wait", + "--timeout", + "120", + ]) + .await + .expect_err("ambiguous policy must report a failed revision"); + assert!( + update_error.contains("ambiguity validation failed"), + "policy update should explain the ambiguity:\n{update_error}" + ); + + let quarantined = guard + .exec(&["python3", "-c", &status_script]) + .await + .expect("exercise both adapters in fail-closed quarantine"); + let quarantined = parse_json_line(&quarantined); + assert_eq!( + quarantined["connect"], 403, + "CONNECT in quarantine: {quarantined}" + ); + assert_eq!( + quarantined["forward"], 403, + "forward in quarantine: {quarantined}" + ); + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + assert_eq!( + server.connection_count(), + connections_before_rejection, + "quarantined requests must not contact the upstream server" + ); + + let logs = run_cli(&[ + "logs", + &guard.name, + "-n", + "500", + "--since", + "2m", + "--source", + "sandbox", + ]) + .await + .expect("fetch sandbox logs after rejection"); + assert!( + logs.contains("configured_mode=fail_closed effective_mode=fail_closed"), + "OCSF logs should state the effective validation posture:\n{logs}" + ); + assert!( + logs.contains("previous policy IS NOT active"), + "OCSF logs should state that the previous policy is inactive:\n{logs}" + ); + assert!( + logs.contains("conflicting metadata") && logs.contains("tls"), + "OCSF logs should contain the overlap rationale:\n{logs}" + ); + + run_cli(&[ + "policy", + "set", + &guard.name, + "--policy", + &valid_policy_path, + "--wait", + "--timeout", + "120", + ]) + .await + .expect("valid policy should exit quarantine"); + let recovered = guard + .exec(&["python3", "-c", &status_script]) + .await + .expect("exercise both adapters after recovery"); + let recovered = parse_json_line(&recovered); + assert_eq!( + recovered["connect"], 200, + "CONNECT after recovery: {recovered}" + ); + assert_eq!( + recovered["forward"], 200, + "forward after recovery: {recovered}" + ); + + guard.cleanup().await; +} + #[tokio::test] async fn destination_denial_modes_match_across_connect_and_forward_adapters() { let policy = write_destination_denial_policy().expect("write destination denial policy"); @@ -856,16 +1144,9 @@ for name, target in targets.items(): print(json.dumps(result, sort_keys=True)) "#; - let guard = SandboxGuard::create(&[ - "--policy", - &policy_path, - "--", - "python3", - "-c", - script, - ]) - .await - .expect("sandbox create"); + let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", script]) + .await + .expect("sandbox create"); let result = parse_json_line(&guard.create_output); for name in [ "metadata", @@ -928,12 +1209,9 @@ async fn explicit_allowed_ips_and_implicit_ip_literals_succeed_through_both_adap let implicit_server = KeepAliveHttpServer::start() .await .expect("start implicit IP-literal server"); - let policy = write_ip_literal_success_policy( - &gateway_ip, - explicit_server.port, - implicit_server.port, - ) - .expect("write IP literal policy"); + let policy = + write_ip_literal_success_policy(&gateway_ip, explicit_server.port, implicit_server.port) + .expect("write IP literal policy"); let policy_path = policy_path(&policy); let mut guard = SandboxGuard::create_keep_with_args( &["--policy", &policy_path], @@ -948,11 +1226,7 @@ async fn explicit_allowed_ips_and_implicit_ip_literals_succeed_through_both_adap ("implicit_ip_literal", implicit_server.port), ] { let output = guard - .exec(&[ - "python3", - "-c", - &proxy_status_script(&gateway_ip, port), - ]) + .exec(&["python3", "-c", &proxy_status_script(&gateway_ip, port)]) .await .unwrap_or_else(|error| panic!("exercise {mode}: {error}")); let statuses = parse_json_line(&output); @@ -1018,6 +1292,261 @@ print("RAW_RELAY_OK") ); } +#[tokio::test] +async fn middleware_redacts_request_bodies_through_both_adapters() { + let server = RequestBodyEchoServer::start() + .await + .expect("start request body echo server"); + let policy = write_middleware_policy(TEST_SERVER_HOST, server.port, "", "fail_closed") + .expect("write middleware policy"); + let policy_path = policy_path(&policy); + let script = format!( + r#" +import json +import os +import socket +import urllib.parse + +HOST = {host:?} +PORT = {port} +SECRET = "sk-1234567890abcdef" + +proxy_url = next( + os.environ[name] + for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") + if os.environ.get(name) +) +parsed = urllib.parse.urlparse(proxy_url) + +def read_response(sock): + data = b"" + while b"\r\n\r\n" not in data: + chunk = sock.recv(4096) + if not chunk: + raise RuntimeError("incomplete response headers") + data += chunk + headers, body = data.split(b"\r\n\r\n", 1) + length = 0 + for line in headers.split(b"\r\n")[1:]: + if line.lower().startswith(b"content-length:"): + length = int(line.split(b":", 1)[1].strip()) + while len(body) < length: + chunk = sock.recv(4096) + if not chunk: + break + body += chunk + status = int(headers.split(None, 2)[1]) + if status != 200: + raise RuntimeError(f"request failed with HTTP {{status}}: {{body!r}}") + return json.loads(body[:length]) + +def request_bytes(target): + body = json.dumps({{"api_key": SECRET}}, separators=(",", ":")).encode() + return ( + f"POST {{target}} HTTP/1.1\r\n" + f"Host: {{HOST}}:{{PORT}}\r\n" + "Content-Type: application/json\r\n" + f"Content-Length: {{len(body)}}\r\n" + "Connection: close\r\n\r\n" + ).encode() + body + +target = f"{{HOST}}:{{PORT}}" +with socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) as forward_sock: + forward_sock.sendall(request_bytes(f"http://{{target}}/middleware")) + forward = read_response(forward_sock) + +with socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) as connect_sock: + connect_sock.sendall(f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode()) + connect_response = b"" + while b"\r\n\r\n" not in connect_response: + connect_response += connect_sock.recv(4096) + if int(connect_response.split(None, 2)[1]) != 200: + raise RuntimeError("CONNECT was denied") + connect_sock.sendall(request_bytes("/middleware")) + connect = read_response(connect_sock) + +print(json.dumps({{"connect": connect, "forward": forward}}, sort_keys=True)) +"#, + host = TEST_SERVER_HOST, + port = server.port, + ); + + let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script]) + .await + .expect("sandbox create"); + let result = parse_json_line(&guard.create_output); + for adapter in ["connect", "forward"] { + assert_eq!( + result[adapter]["api_key"], "[REDACTED]", + "{adapter} did not deliver the middleware-transformed body: {result}" + ); + } +} + +#[tokio::test] +async fn fail_closed_middleware_blocks_uninspectable_connect_payload_before_upstream() { + let server = EchoServer::start().await.expect("start TCP echo server"); + let policy = write_middleware_policy(TEST_SERVER_HOST, server.port, "", "fail_closed") + .expect("write fail-closed middleware policy"); + let policy_path = policy_path(&policy); + let script = format!( + r#" +import os +import socket +import urllib.parse + +HOST = {host:?} +PORT = {port} +PAYLOAD = bytes([0x00, 0xff, 0x13, 0x37]) + b"not-http-or-tls" + +proxy_url = next( + os.environ[name] + for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") + if os.environ.get(name) +) +parsed = urllib.parse.urlparse(proxy_url) +with socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) as sock: + target = f"{{HOST}}:{{PORT}}" + sock.sendall(f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode()) + response = b"" + while b"\r\n\r\n" not in response: + response += sock.recv(4096) + if int(response.split(None, 2)[1]) != 200: + raise RuntimeError("CONNECT was denied before tunnel establishment") + sock.sendall(PAYLOAD) + denial = b"" + while True: + try: + chunk = sock.recv(4096) + except ConnectionResetError: + break + if not chunk: + break + denial += chunk + if denial and ( + b"HTTP/1.1 403 Forbidden" not in denial + or b"unsupported_l7_protocol" not in denial + ): + raise RuntimeError(f"missing fail-closed middleware denial: {{denial!r}}") +print("UNINSPECTABLE_MIDDLEWARE_BLOCKED") +"#, + host = TEST_SERVER_HOST, + port = server.port, + ); + + let mut guard = SandboxGuard::create_keep_with_args( + &["--policy", &policy_path], + &["sh", "-c", "echo Ready; sleep infinity"], + "Ready", + ) + .await + .expect("create keep sandbox"); + let output = guard + .exec(&["python3", "-c", &script]) + .await + .expect("exercise uninspectable fail-closed middleware"); + assert!( + output.contains("UNINSPECTABLE_MIDDLEWARE_BLOCKED"), + "uninspectable payload was not blocked:\n{output}" + ); + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + assert!( + server.observed_bytes().is_empty(), + "uninspectable payload reached upstream before middleware denial" + ); + + let logs = run_cli(&[ + "logs", + &guard.name, + "-n", + "500", + "--since", + "2m", + "--source", + "sandbox", + ]) + .await + .expect("fetch sandbox logs after middleware denial"); + assert!( + logs.contains("openshell.middleware.traffic_uninspectable") + && logs + .contains("Unsupported tunnel protocol cannot be inspected by required middleware"), + "OCSF logs should explain the fail-closed denial:\n{logs}" + ); + + guard.cleanup().await; +} + +#[tokio::test] +async fn fail_open_middleware_bypasses_uninspectable_tls_skip_connect() { + let server = EchoServer::start().await.expect("start TCP echo server"); + let policy = write_middleware_policy( + TEST_SERVER_HOST, + server.port, + " tls: skip", + "fail_open", + ) + .expect("write fail-open middleware policy"); + let policy_path = policy_path(&policy); + let script = format!( + r#" +import os +import socket +import urllib.parse + +HOST = {host:?} +PORT = {port} +PAYLOAD = bytes([0x00, 0xff, 0x13, 0x37, 0x80]) + b"middleware-bypass" + +proxy_url = next( + os.environ[name] + for name in ("HTTP_PROXY", "http_proxy", "HTTPS_PROXY", "https_proxy") + if os.environ.get(name) +) +parsed = urllib.parse.urlparse(proxy_url) +with socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) as sock: + target = f"{{HOST}}:{{PORT}}" + sock.sendall(f"CONNECT {{target}} HTTP/1.1\r\nHost: {{target}}\r\n\r\n".encode()) + response = b"" + while b"\r\n\r\n" not in response: + response += sock.recv(4096) + if int(response.split(None, 2)[1]) != 200: + raise RuntimeError("CONNECT was denied") + sock.sendall(PAYLOAD) + echoed = b"" + while len(echoed) < len(PAYLOAD): + chunk = sock.recv(len(PAYLOAD) - len(echoed)) + if not chunk: + break + echoed += chunk + if echoed != PAYLOAD: + raise RuntimeError(f"fail-open middleware did not preserve raw relay: {{echoed!r}}") +print("UNINSPECTABLE_MIDDLEWARE_BYPASSED") +"#, + host = TEST_SERVER_HOST, + port = server.port, + ); + + let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script]) + .await + .expect("sandbox create"); + assert!( + guard + .create_output + .contains("UNINSPECTABLE_MIDDLEWARE_BYPASSED"), + "fail-open middleware did not bypass uninspectable traffic:\n{}", + guard.create_output + ); + assert_eq!( + server.observed_bytes(), + [0x00, 0xff, 0x13, 0x37, 0x80] + .into_iter() + .chain(*b"middleware-bypass") + .collect::>(), + "upstream did not receive the unchanged fail-open payload" + ); +} + #[tokio::test] async fn forward_pipeline_never_reaches_upstream_as_first_request_overflow() { let server = PipelineProbeServer::start() @@ -1069,16 +1598,9 @@ print("FORWARD_PIPELINE_CLOSED") port = server.port, ); - let guard = SandboxGuard::create(&[ - "--policy", - &policy_path, - "--", - "python3", - "-c", - &script, - ]) - .await - .expect("sandbox create"); + let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script]) + .await + .expect("sandbox create"); assert!( guard.create_output.contains("FORWARD_PIPELINE_CLOSED"), "forward proxy did not close after one response:\n{}", @@ -1087,7 +1609,10 @@ print("FORWARD_PIPELINE_CLOSED") let observed = String::from_utf8(server.observed_request()).expect("upstream HTTP request"); assert!(observed.starts_with("GET /allowed HTTP/1.1\r\n")); - assert!(observed.contains("Connection: close\r\n")); + assert!( + !observed.to_ascii_lowercase().contains("\r\nconnection:"), + "shared relay must remove hop-by-hop connection headers:\n{observed}" + ); assert!(!observed.contains("/blocked")); } From c49a86cb3756f2bc1e1f6bda1a6c2ff58414ce2d Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Mon, 20 Jul 2026 10:26:34 -0700 Subject: [PATCH 12/30] fix(config): move policy failure mode to gateway toml Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- architecture/security-policy.md | 7 +- crates/openshell-core/src/config.rs | 64 ++++++++++++- crates/openshell-core/src/grpc_client.rs | 37 ++++++++ crates/openshell-core/src/lib.rs | 2 +- crates/openshell-core/src/settings.rs | 42 --------- crates/openshell-sandbox/src/lib.rs | 94 ++----------------- crates/openshell-server/src/cli.rs | 14 +++ crates/openshell-server/src/config_file.rs | 20 ++++ crates/openshell-server/src/grpc/policy.rs | 73 +++++++++----- deploy/helm/openshell/README.md | 1 + .../openshell/templates/gateway-config.yaml | 5 + .../openshell/tests/gateway_config_test.yaml | 16 ++++ deploy/helm/openshell/values.yaml | 3 + docs/reference/gateway-config.mdx | 6 ++ docs/sandboxes/policies.mdx | 18 ++-- proto/sandbox.proto | 4 + 16 files changed, 233 insertions(+), 173 deletions(-) diff --git a/architecture/security-policy.md b/architecture/security-policy.md index 5e18b0d176..a765994602 100644 --- a/architecture/security-policy.md +++ b/architecture/security-policy.md @@ -104,13 +104,14 @@ deny rules only when their connection and request-processing metadata agree; conflicting TLS, destination, credential, parser, or enforcement metadata rejects the complete generation. -The gateway-global `policy_validation_failure_mode` setting controls rejected -generations. It defaults to `fail_closed`, which publishes a quarantine +The `[openshell.gateway] policy_validation_failure_mode` configuration controls +rejected generations. It defaults to `fail_closed`, which publishes a quarantine generation, denies new egress, invalidates existing relays, and leaves the previous policy inactive. Operators may explicitly select `retain_last_valid`, which keeps the previous generation active. With no previous valid generation, the effective mode remains `fail_closed` regardless -of the configured mode. OCSF configuration and finding events state the +of the configured mode. The gateway distributes this startup configuration to +sandbox supervisors with each effective policy snapshot. OCSF configuration and finding events state the candidate version, validation rationale, configured and effective modes, active generation, and whether the previous policy is active. Static controls, such as filesystem allowlists and process identity, require a new sandbox diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index b200eded64..daa867f16f 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -36,6 +36,43 @@ pub const DEFAULT_DOCKER_NETWORK_NAME: &str = "openshell-docker"; /// Default domain used for browser-facing sandbox service URLs. pub const DEFAULT_SERVICE_ROUTING_DOMAIN: &str = "openshell.localhost"; +/// Gateway posture when a sandbox rejects a candidate policy generation. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PolicyValidationFailureMode { + /// Deactivate the previous policy and deny new egress until a valid + /// generation is loaded. + #[default] + FailClosed, + /// Keep the last valid generation active when a newer candidate fails + /// validation. Startup still fails closed when no valid generation exists. + RetainLastValid, +} + +impl PolicyValidationFailureMode { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::FailClosed => "fail_closed", + Self::RetainLastValid => "retain_last_valid", + } + } +} + +impl FromStr for PolicyValidationFailureMode { + type Err = String; + + fn from_str(value: &str) -> Result { + match value { + "fail_closed" => Ok(Self::FailClosed), + "retain_last_valid" => Ok(Self::RetainLastValid), + _ => Err(format!( + "invalid policy validation failure mode '{value}'; expected fail_closed or retain_last_valid" + )), + } + } +} + /// Default OCI repository for the supervisor image (no tag). pub const DEFAULT_SUPERVISOR_IMAGE_REPO: &str = "ghcr.io/nvidia/openshell/supervisor"; @@ -396,6 +433,9 @@ pub struct Config { /// Log level (trace, debug, info, warn, error). pub log_level: String, + /// Security posture for rejected sandbox policy generations. + pub policy_validation_failure_mode: PolicyValidationFailureMode, + /// TLS configuration. When `None`, the server listens on plaintext HTTP. pub tls: Option, @@ -737,6 +777,7 @@ impl Config { health_bind_address: None, metrics_bind_address: None, log_level: default_log_level(), + policy_validation_failure_mode: PolicyValidationFailureMode::default(), tls, oidc: None, auth: GatewayAuthConfig::default(), @@ -981,10 +1022,10 @@ mod tests { use super::{ ComputeDriverKind, Config, DEFAULT_SERVICE_ROUTING_DOMAIN, GatewayInterceptorBindingPolicy, GatewayInterceptorConfig, GatewayInterceptorFailurePolicy, GatewayJwtConfig, - GatewayProviderProfileSourceConfig, detect_docker_socket_from_candidates, detect_driver, - detect_podman_socket_from_candidates, docker_host_unix_socket_path, docker_socket_responds, - is_unix_socket, normalize_compute_driver_name, podman_socket_candidates_from_env, - podman_socket_responds, + GatewayProviderProfileSourceConfig, PolicyValidationFailureMode, + detect_docker_socket_from_candidates, detect_driver, detect_podman_socket_from_candidates, + docker_host_unix_socket_path, docker_socket_responds, is_unix_socket, + normalize_compute_driver_name, podman_socket_candidates_from_env, podman_socket_responds, }; #[cfg(unix)] use std::io::{Read as _, Write as _}; @@ -1020,6 +1061,21 @@ mod tests { assert!(err.contains("unsupported compute driver 'firecracker'")); } + #[test] + fn policy_validation_failure_mode_is_secure_by_default() { + assert_eq!( + Config::new(None).policy_validation_failure_mode, + PolicyValidationFailureMode::FailClosed + ); + assert_eq!( + "retain_last_valid" + .parse::() + .unwrap(), + PolicyValidationFailureMode::RetainLastValid + ); + assert!("keep_old".parse::().is_err()); + } + #[test] fn compute_driver_name_normalization_accepts_builtin_and_custom_names() { assert_eq!(normalize_compute_driver_name(" VM ").unwrap(), "vm"); diff --git a/crates/openshell-core/src/grpc_client.rs b/crates/openshell-core/src/grpc_client.rs index 070704fb0a..579ee4a5b3 100644 --- a/crates/openshell-core/src/grpc_client.rs +++ b/crates/openshell-core/src/grpc_client.rs @@ -780,6 +780,8 @@ pub struct SettingsPollResult { pub supervisor_middleware_services: Vec, /// Workspace the sandbox belongs to. pub workspace: String, + /// Gateway-configured posture for rejected policy generations. + pub policy_validation_failure_mode: crate::PolicyValidationFailureMode, } fn settings_poll_result(inner: crate::proto::GetSandboxConfigResponse) -> SettingsPollResult { @@ -795,6 +797,41 @@ fn settings_poll_result(inner: crate::proto::GetSandboxConfigResponse) -> Settin provider_env_revision: inner.provider_env_revision, supervisor_middleware_services: inner.supervisor_middleware_services, workspace: inner.workspace, + policy_validation_failure_mode: inner + .policy_validation_failure_mode + .parse() + .unwrap_or_default(), + } +} + +#[cfg(test)] +mod settings_poll_tests { + use super::settings_poll_result; + use crate::PolicyValidationFailureMode; + use crate::proto::GetSandboxConfigResponse; + + #[test] + fn validation_failure_mode_round_trips_from_gateway_config() { + let result = settings_poll_result(GetSandboxConfigResponse { + policy_validation_failure_mode: "retain_last_valid".to_string(), + ..Default::default() + }); + assert_eq!( + result.policy_validation_failure_mode, + PolicyValidationFailureMode::RetainLastValid + ); + } + + #[test] + fn unknown_validation_failure_mode_fails_closed() { + let result = settings_poll_result(GetSandboxConfigResponse { + policy_validation_failure_mode: "future_mode".to_string(), + ..Default::default() + }); + assert_eq!( + result.policy_validation_failure_mode, + PolicyValidationFailureMode::FailClosed + ); } } diff --git a/crates/openshell-core/src/lib.rs b/crates/openshell-core/src/lib.rs index 80bfbb046d..56ffda38c4 100644 --- a/crates/openshell-core/src/lib.rs +++ b/crates/openshell-core/src/lib.rs @@ -45,7 +45,7 @@ pub use config::{ ComputeDriverKind, Config, GatewayAuthConfig, GatewayInterceptorBindingOverride, GatewayInterceptorBindingPolicy, GatewayInterceptorConfig, GatewayInterceptorFailurePolicy, GatewayInterceptorPhaseConfig, GatewayJwtConfig, GatewayProviderProfileSourceConfig, - MtlsAuthConfig, OidcConfig, TlsConfig, + MtlsAuthConfig, OidcConfig, PolicyValidationFailureMode, TlsConfig, }; pub use error::{ComputeDriverError, Error, Result}; pub use metadata::{ diff --git a/crates/openshell-core/src/settings.rs b/crates/openshell-core/src/settings.rs index 4ae59272b0..156e4c3845 100644 --- a/crates/openshell-core/src/settings.rs +++ b/crates/openshell-core/src/settings.rs @@ -107,16 +107,6 @@ pub const PROPOSAL_APPROVAL_MODE_KEY: &str = "proposal_approval_mode"; /// fail-closes on unknown persisted values for defense in depth. pub const PROPOSAL_APPROVAL_MODE_VALUES: &[&str] = &["manual", "auto"]; -/// Gateway security posture when a sandbox rejects a candidate policy during -/// validation. The default is [`POLICY_VALIDATION_FAILURE_MODE_FAIL_CLOSED`]. -pub const POLICY_VALIDATION_FAILURE_MODE_KEY: &str = "policy_validation_failure_mode"; -pub const POLICY_VALIDATION_FAILURE_MODE_FAIL_CLOSED: &str = "fail_closed"; -pub const POLICY_VALIDATION_FAILURE_MODE_RETAIN_LAST_VALID: &str = "retain_last_valid"; -pub const POLICY_VALIDATION_FAILURE_MODE_VALUES: &[&str] = &[ - POLICY_VALIDATION_FAILURE_MODE_FAIL_CLOSED, - POLICY_VALIDATION_FAILURE_MODE_RETAIN_LAST_VALID, -]; - pub const REGISTERED_SETTINGS: &[RegisteredSetting] = &[ // Gateway-level opt-in for provider profile policy composition. Defaults // to false when unset. @@ -147,14 +137,6 @@ pub const REGISTERED_SETTINGS: &[RegisteredSetting] = &[ kind: SettingValueKind::String, allowed_string_values: Some(PROPOSAL_APPROVAL_MODE_VALUES), }, - // Gateway security posture for policy validation failures. Defaults to - // fail_closed when unset. Operators may explicitly select availability- - // oriented last-known-good retention. - RegisteredSetting { - key: POLICY_VALIDATION_FAILURE_MODE_KEY, - kind: SettingValueKind::String, - allowed_string_values: Some(POLICY_VALIDATION_FAILURE_MODE_VALUES), - }, ]; /// Resolve a setting descriptor from the registry by key. @@ -186,8 +168,6 @@ pub fn parse_bool_like(raw: &str) -> Option { #[cfg(test)] mod tests { use super::{ - POLICY_VALIDATION_FAILURE_MODE_FAIL_CLOSED, POLICY_VALIDATION_FAILURE_MODE_KEY, - POLICY_VALIDATION_FAILURE_MODE_RETAIN_LAST_VALID, POLICY_VALIDATION_FAILURE_MODE_VALUES, PROPOSAL_APPROVAL_MODE_KEY, PROPOSAL_APPROVAL_MODE_VALUES, PROVIDERS_V2_ENABLED_KEY, REGISTERED_SETTINGS, RegisteredSetting, SettingValueKind, parse_bool_like, registered_keys_csv, setting_for_key, @@ -257,28 +237,6 @@ mod tests { } } - #[test] - fn policy_validation_failure_mode_accepts_only_documented_values() { - let setting = setting_for_key(POLICY_VALIDATION_FAILURE_MODE_KEY) - .expect("policy validation failure mode should be registered"); - assert_eq!(setting.kind, SettingValueKind::String); - assert_eq!( - setting.allowed_string_values, - Some(POLICY_VALIDATION_FAILURE_MODE_VALUES) - ); - assert!( - setting - .validate_string_value(POLICY_VALIDATION_FAILURE_MODE_FAIL_CLOSED) - .is_ok() - ); - assert!( - setting - .validate_string_value(POLICY_VALIDATION_FAILURE_MODE_RETAIN_LAST_VALID) - .is_ok() - ); - assert!(setting.validate_string_value("keep_old").is_err()); - } - // ---- parse_bool_like ---- #[test] diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index e55014a98e..668d1c6c56 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -23,6 +23,8 @@ use std::sync::atomic::{AtomicBool, AtomicU32}; use std::time::Duration; use tracing::{debug, info, warn}; +use openshell_core::PolicyValidationFailureMode; + use openshell_ocsf::{ ActionId, ActivityId, AppLifecycleBuilder, ConfigStateChangeBuilder, DetectionFindingBuilder, DispositionId, FindingInfo, OcsfEvent, SandboxContext, SeverityId, StateId, StatusId, @@ -2024,7 +2026,7 @@ async fn load_policy( let engine = Arc::new(OpaEngine::from_proto(&proto_policy)?); let disposition = apply_policy_validation_failure( &engine, - PolicyValidationFailureMode::from_settings(&snapshot.settings), + snapshot.policy_validation_failure_mode, has_last_valid_policy, candidate_version, &validation_error, @@ -2635,39 +2637,6 @@ async fn reconcile_middleware_registry( } } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum PolicyValidationFailureMode { - FailClosed, - RetainLastValid, -} - -impl PolicyValidationFailureMode { - fn from_settings( - settings: &std::collections::HashMap, - ) -> Self { - match extract_string_setting( - settings, - openshell_core::settings::POLICY_VALIDATION_FAILURE_MODE_KEY, - ) { - Some(openshell_core::settings::POLICY_VALIDATION_FAILURE_MODE_RETAIN_LAST_VALID) => { - Self::RetainLastValid - } - _ => Self::FailClosed, - } - } - - const fn as_str(self) -> &'static str { - match self { - Self::FailClosed => { - openshell_core::settings::POLICY_VALIDATION_FAILURE_MODE_FAIL_CLOSED - } - Self::RetainLastValid => { - openshell_core::settings::POLICY_VALIDATION_FAILURE_MODE_RETAIN_LAST_VALID - } - } - } -} - #[derive(Debug, PartialEq, Eq)] struct PolicyValidationFailureDisposition { configured_mode: PolicyValidationFailureMode, @@ -2972,7 +2941,7 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { // fail-closed quarantine, so an explicit retain_last_valid selection // can reactivate it without accepting any part of the invalid policy. if !policy_changed && let Some(rejected) = rejected_policy_generation.as_mut() { - let mode = PolicyValidationFailureMode::from_settings(&result.settings); + let mode = result.policy_validation_failure_mode; if mode != rejected.configured_mode { let disposition = apply_policy_validation_failure( &ctx.opa_engine, @@ -3189,8 +3158,7 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { )) .build()); - let failure_mode = - PolicyValidationFailureMode::from_settings(&result.settings); + let failure_mode = result.policy_validation_failure_mode; let disposition = apply_policy_validation_failure( &ctx.opa_engine, failure_mode, @@ -3331,22 +3299,6 @@ fn apply_agent_proposals_enabled( } } -/// Extract a string value from an effective setting, if present. -fn extract_string_setting<'a>( - settings: &'a std::collections::HashMap, - key: &str, -) -> Option<&'a str> { - use openshell_core::proto::setting_value; - settings - .get(key) - .and_then(|es| es.value.as_ref()) - .and_then(|sv| sv.value.as_ref()) - .and_then(|value| match value { - setting_value::Value::StringValue(value) => Some(value.as_str()), - _ => None, - }) -} - /// Log individual setting changes between two snapshots. fn log_setting_changes( old: &std::collections::HashMap, @@ -3772,6 +3724,7 @@ filesystem_policy: provider_env_revision: 0, supervisor_middleware_services: Vec::new(), workspace: String::new(), + policy_validation_failure_mode: PolicyValidationFailureMode::default(), } } @@ -4071,41 +4024,6 @@ filesystem_policy: "workspace must survive the snapshot so sync_policy_and_fetch_snapshot receives it" ); } - - fn string_effective_setting(value: &str) -> openshell_core::proto::EffectiveSetting { - openshell_core::proto::EffectiveSetting { - value: Some(openshell_core::proto::SettingValue { - value: Some(openshell_core::proto::setting_value::Value::StringValue( - value.to_string(), - )), - }), - scope: openshell_core::proto::SettingScope::Global.into(), - } - } - - #[test] - fn policy_validation_failure_mode_defaults_to_fail_closed() { - let settings = std::collections::HashMap::new(); - assert_eq!( - PolicyValidationFailureMode::from_settings(&settings), - PolicyValidationFailureMode::FailClosed - ); - } - - #[test] - fn policy_validation_failure_mode_requires_explicit_retain_setting() { - let settings = std::collections::HashMap::from([( - openshell_core::settings::POLICY_VALIDATION_FAILURE_MODE_KEY.to_string(), - string_effective_setting( - openshell_core::settings::POLICY_VALIDATION_FAILURE_MODE_RETAIN_LAST_VALID, - ), - )]); - assert_eq!( - PolicyValidationFailureMode::from_settings(&settings), - PolicyValidationFailureMode::RetainLastValid - ); - } - #[test] fn fail_closed_validation_failure_deactivates_previous_generation() { let engine = OpaEngine::from_strings( diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index 45fc3fc2db..8b18034947 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -404,6 +404,13 @@ fn prepare_server_config(args: &mut RunArgs, matches: &ArgMatches) -> Result, #[serde(default)] pub grpc_rate_limit_window_seconds: Option, + /// Security posture when a sandbox rejects a candidate policy generation. + #[serde(default)] + pub policy_validation_failure_mode: Option, // ── Service routing ────────────────────────────────────────────────── /// Subject Alternative Names configured on the gateway server certificate. @@ -422,6 +425,7 @@ compute_drivers = ["kubernetes"] sandbox_namespace = "agents" grpc_rate_limit_requests = 120 grpc_rate_limit_window_seconds = 60 +policy_validation_failure_mode = "retain_last_valid" default_image = "ghcr.io/nvidia/openshell/sandbox:latest" supervisor_image = "ghcr.io/nvidia/openshell/supervisor:latest" client_tls_secret_name = "openshell-sandbox-tls" @@ -450,6 +454,10 @@ grpc_endpoint = "https://openshell-gateway.agents.svc:8080" ); assert_eq!(gw.grpc_rate_limit_requests, Some(120)); assert_eq!(gw.grpc_rate_limit_window_seconds, Some(60)); + assert_eq!( + gw.policy_validation_failure_mode, + Some(openshell_core::PolicyValidationFailureMode::RetainLastValid) + ); assert!(gw.tls.is_some()); assert!(gw.oidc.is_some()); assert!(file.openshell.drivers.contains_key("kubernetes")); @@ -513,6 +521,18 @@ sampler = "traceidratio" ); } + #[test] + fn rejects_unknown_policy_validation_failure_mode() { + let tmp = write_tmp( + r#" +[openshell.gateway] +policy_validation_failure_mode = "keep_old" +"#, + ); + let error = load(tmp.path()).expect_err("unknown posture must fail TOML validation"); + assert!(error.to_string().contains("policy_validation_failure_mode")); + } + #[test] fn parses_gateway_auth_config() { let toml = r" diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index ee02393638..05ac8d5feb 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -1423,11 +1423,12 @@ pub(super) async fn handle_get_sandbox_config( let settings = merge_effective_settings(&global_settings, &sandbox_settings)?; let supervisor_middleware_services = state.middleware_registry.required_services(policy.as_ref()); - let config_revision = compute_config_revision( + let config_revision = compute_config_revision_with_validation_mode( policy.as_ref(), &settings, policy_source, &supervisor_middleware_services, + state.config.policy_validation_failure_mode, ); let provider_env_revision = compute_provider_env_revision_with_catalog( state.store.as_ref(), @@ -1448,6 +1449,11 @@ pub(super) async fn handle_get_sandbox_config( provider_env_revision, supervisor_middleware_services, workspace, + policy_validation_failure_mode: state + .config + .policy_validation_failure_mode + .as_str() + .to_string(), })) } @@ -1731,10 +1737,6 @@ async fn handle_update_config_inner( "one of policy, setting_key, or merge_operations must be provided", )); } - if has_setting { - validate_setting_scope(key, req.global)?; - } - if req.global { if !req.annotations.is_empty() { return Err(Status::invalid_argument( @@ -3695,14 +3697,16 @@ fn deterministic_policy_hash(policy: &ProtoSandboxPolicy) -> String { } /// Compute a fingerprint for the effective sandbox configuration. -fn compute_config_revision( +fn compute_config_revision_with_validation_mode( policy: Option<&ProtoSandboxPolicy>, settings: &HashMap, policy_source: PolicySource, supervisor_middleware_services: &[openshell_core::proto::SupervisorMiddlewareService], + policy_validation_failure_mode: openshell_core::PolicyValidationFailureMode, ) -> u64 { let mut hasher = Sha256::new(); hasher.update((policy_source as i32).to_le_bytes()); + hasher.update(policy_validation_failure_mode.as_str().as_bytes()); if let Some(policy) = policy { hasher.update(deterministic_policy_hash(policy).as_bytes()); } @@ -3744,6 +3748,22 @@ fn compute_config_revision( u64::from_le_bytes(bytes) } +#[cfg(test)] +fn compute_config_revision( + policy: Option<&ProtoSandboxPolicy>, + settings: &HashMap, + policy_source: PolicySource, + supervisor_middleware_services: &[openshell_core::proto::SupervisorMiddlewareService], +) -> u64 { + compute_config_revision_with_validation_mode( + policy, + settings, + policy_source, + supervisor_middleware_services, + openshell_core::PolicyValidationFailureMode::default(), + ) +} + fn decode_draft_chunk_rule(record: &DraftChunkRecord) -> Result, Status> { if record.proposed_rule.is_empty() { Ok(None) @@ -4326,16 +4346,6 @@ fn validate_registered_setting_key( }) } -fn validate_setting_scope(key: &str, global: bool) -> Result<(), Status> { - if key == settings::POLICY_VALIDATION_FAILURE_MODE_KEY && !global { - return Err(Status::invalid_argument(format!( - "setting '{}' is gateway-global and must be configured with --global", - settings::POLICY_VALIDATION_FAILURE_MODE_KEY - ))); - } - Ok(()) -} - fn proto_setting_to_stored(key: &str, value: &SettingValue) -> Result { let setting = validate_registered_setting_key(key)?; let expected = setting.kind; @@ -11757,6 +11767,28 @@ mod tests { assert_ne!(rev_a, rev_b); } + #[test] + fn config_revision_changes_when_validation_failure_mode_changes() { + let policy = ProtoSandboxPolicy::default(); + let settings = HashMap::new(); + + let fail_closed = compute_config_revision_with_validation_mode( + Some(&policy), + &settings, + PolicySource::Sandbox, + &[], + openshell_core::PolicyValidationFailureMode::FailClosed, + ); + let retain_last_valid = compute_config_revision_with_validation_mode( + Some(&policy), + &settings, + PolicySource::Sandbox, + &[], + openshell_core::PolicyValidationFailureMode::RetainLastValid, + ); + assert_ne!(fail_closed, retain_last_valid); + } + #[test] fn config_revision_changes_when_supervisor_middleware_services_change() { let policy = ProtoSandboxPolicy::default(); @@ -12234,15 +12266,6 @@ mod tests { assert!(err.message().contains("unknown setting key")); } - #[test] - fn policy_validation_failure_mode_is_gateway_global_only() { - assert!(validate_setting_scope(settings::POLICY_VALIDATION_FAILURE_MODE_KEY, true).is_ok()); - let error = validate_setting_scope(settings::POLICY_VALIDATION_FAILURE_MODE_KEY, false) - .expect_err("sandbox-scoped override must be rejected"); - assert_eq!(error.code(), Code::InvalidArgument); - assert!(error.message().contains("must be configured with --global")); - } - #[test] fn proto_setting_to_stored_rejects_policy_key() { let value = SettingValue { diff --git a/deploy/helm/openshell/README.md b/deploy/helm/openshell/README.md index d29873c6fc..d4310cb9a7 100644 --- a/deploy/helm/openshell/README.md +++ b/deploy/helm/openshell/README.md @@ -214,6 +214,7 @@ add `ci/values-spire.yaml` to the OpenShell release values files. | server.oidc.rolesClaim | string | `""` | Dot-separated path to the roles array in the JWT claims. Keycloak: "realm_access.roles", Entra ID: "roles", Okta: "groups". | | server.oidc.scopesClaim | string | `""` | Dot-separated path to the scopes array in the JWT claims. | | server.oidc.userRole | string | `""` | Role name for standard user access. | +| server.policyValidationFailureMode | string | `"fail_closed"` | Posture when a candidate sandbox policy fails validation. `fail_closed` deactivates the previous policy; `retain_last_valid` keeps it active. | | server.providerTokenGrants.spiffe.enabled | bool | `false` | Mount the SPIFFE Workload API socket into sandbox pods for dynamic provider token grants. | | server.providerTokenGrants.spiffe.workloadApiSocketPath | string | `"/spiffe-workload-api/spire-agent.sock"` | Path to the SPIFFE Workload API socket mounted into sandbox pods. | | server.sandboxImage | string | `"ghcr.io/nvidia/openshell-community/sandboxes/base:latest"` | Default sandbox image used when requests do not specify one. | diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 41cadfbad4..0c2fc3bbd4 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -33,6 +33,11 @@ data: {{- end }} log_level = {{ .Values.server.logLevel | quote }} sandbox_namespace = {{ include "openshell.sandboxNamespace" . | quote }} + {{- $policyValidationFailureMode := .Values.server.policyValidationFailureMode }} + {{- if not (has $policyValidationFailureMode (list "fail_closed" "retain_last_valid")) }} + {{- fail "server.policyValidationFailureMode must be fail_closed or retain_last_valid" }} + {{- end }} + policy_validation_failure_mode = {{ $policyValidationFailureMode | quote }} default_image = {{ .Values.server.sandboxImage | quote }} {{- if include "openshell.supervisorImageOverrideEnabled" . }} supervisor_image = {{ include "openshell.supervisorImage" . | quote }} diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index aee396c38f..90e4f9cef0 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -229,6 +229,22 @@ tests: path: data["gateway.toml"] pattern: 'grpc_rate_limit_window_seconds\s*=' + - it: renders fail-closed policy validation posture by default + template: templates/gateway-config.yaml + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.gateway\].*?policy_validation_failure_mode\s*=\s*"fail_closed"' + + - it: renders retain-last-valid policy validation posture + template: templates/gateway-config.yaml + set: + server.policyValidationFailureMode: retain_last_valid + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.gateway\].*?policy_validation_failure_mode\s*=\s*"retain_last_valid"' + - it: renders the gRPC rate limit under [openshell.gateway] when both values are positive template: templates/gateway-config.yaml set: diff --git a/deploy/helm/openshell/values.yaml b/deploy/helm/openshell/values.yaml index 8abd3dea71..0525ed475d 100644 --- a/deploy/helm/openshell/values.yaml +++ b/deploy/helm/openshell/values.yaml @@ -229,6 +229,9 @@ server: # -- Enable plaintext HTTP routing for loopback sandbox service URLs on # TLS-enabled gateways. enableLoopbackServiceHttp: true + # -- Posture when a candidate sandbox policy fails validation. `fail_closed` + # deactivates the previous policy; `retain_last_valid` keeps it active. + policyValidationFailureMode: fail_closed # Optional gateway-wide gRPC request rate limit. Applies only to gRPC API # traffic after protocol multiplexing; health, metrics, and loopback service # HTTP routes are not rate limited. Both values must be positive to enable the diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index d81aaa95f6..69a1325c58 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -75,6 +75,10 @@ compute_drivers = ["kubernetes"] sandbox_namespace = "openshell" ssh_session_ttl_secs = 3600 +# Reject invalid policy generations securely by default. Set +# "retain_last_valid" only when availability takes priority. +policy_validation_failure_mode = "fail_closed" + # Subject Alternative Names baked into the gateway server certificate. # Wildcard DNS SANs (e.g. "*.dev.openshell.localhost") also enable sandbox # service URLs under that domain. @@ -176,6 +180,8 @@ phases = ["validate"] Local Docker, Podman, and VM gateways can also set `[openshell.gateway.mtls_auth] enabled = true` to authenticate CLI callers from verified client certificates. Kubernetes deployments must leave this unset and use OIDC or a trusted access proxy; the Helm chart does not render this table. +`[openshell.gateway] policy_validation_failure_mode` controls what sandbox supervisors do when a complete candidate policy fails validation. The default, `fail_closed`, deactivates the previous network policy, closes relays pinned to it, and denies new egress until a valid generation loads. `retain_last_valid` leaves the previous valid generation active. Both modes reject the candidate atomically; startup always fails closed when no previous valid generation exists. Changing the value requires restarting the gateway so it can reload `gateway.toml` and distribute the new posture to sandbox supervisors. + `[openshell.gateway.gateway_jwt] ttl_secs` controls gateway-minted sandbox JWT lifetime. When omitted, it defaults to `0`: the token `exp` claim and `expires_at_ms` response field become `0`, and the sandbox JWT does not expire. Use that default only for local single-player Docker, Podman, or VM gateways. Kubernetes and other shared deployments should set a positive TTL; Helm renders `3600` seconds by default, and the gateway logs a warning when a Kubernetes gateway uses `0`. `[openshell.gateway.auth] allow_unauthenticated_users = true` is an unsafe local-development and trusted-proxy escape hatch. It accepts user-facing CLI/API calls without OIDC or mTLS credentials while sandbox supervisors still authenticate with gateway-minted sandbox JWTs. Leave it false for shared and production gateways. diff --git a/docs/sandboxes/policies.mdx b/docs/sandboxes/policies.mdx index 2f6150c962..fce3841d2a 100644 --- a/docs/sandboxes/policies.mdx +++ b/docs/sandboxes/policies.mdx @@ -205,25 +205,23 @@ The following steps outline the hot-reload policy update workflow. OpenShell validates a complete candidate policy before activating any part of it. Endpoints may overlap when their connection and request-processing metadata agree. For example, two `api.example.com:443` REST entries can contribute different allow and deny rules when they use the same TLS, destination, credential, parser, and enforcement settings. OpenShell rejects the candidate when overlapping exact or wildcard host selectors disagree on those fields. Path-scoped endpoints may use different protocols only when their path selectors cannot match the same request. -The gateway-global `policy_validation_failure_mode` setting determines what happens after rejection. Its default is `fail_closed`: +The gateway's `policy_validation_failure_mode` configuration determines what happens after rejection. Set it under `[openshell.gateway]` in `gateway.toml`. Its default is `fail_closed`: -```shell -openshell settings set --global \ - --key policy_validation_failure_mode \ - --value fail_closed +```toml +[openshell.gateway] +policy_validation_failure_mode = "fail_closed" ``` In `fail_closed` mode, the supervisor publishes a quarantine generation, denies new egress, and closes connections pinned to the previous generation. The previous policy is not active. A later valid policy exits quarantine automatically. Operators that explicitly prioritize availability can retain the previous generation: -```shell -openshell settings set --global \ - --key policy_validation_failure_mode \ - --value retain_last_valid +```toml +[openshell.gateway] +policy_validation_failure_mode = "retain_last_valid" ``` -In `retain_last_valid` mode, the rejected candidate remains inactive and the previous valid generation remains active. If no previous valid generation exists, such as during initial startup, OpenShell still fails closed. The setting is gateway-global and cannot be overridden by an individual sandbox. +In `retain_last_valid` mode, the rejected candidate remains inactive and the previous valid generation remains active. If no previous valid generation exists, such as during initial startup, OpenShell still fails closed. Restart the gateway after changing `gateway.toml`; connected sandbox supervisors receive the configured posture from the restarted gateway. Individual sandboxes cannot override it. OCSF configuration and finding events identify the rejected candidate, validation rationale, configured and effective modes, active generation, and whether the previous policy is active. When `retain_last_valid` is configured without a previous valid generation, the effective mode remains `fail_closed`. Connection denials during quarantine include the validation failure as their policy denial rationale. diff --git a/proto/sandbox.proto b/proto/sandbox.proto index 16b3ca998d..9ccefadefb 100644 --- a/proto/sandbox.proto +++ b/proto/sandbox.proto @@ -367,6 +367,10 @@ message GetSandboxConfigResponse { // Workspace the sandbox belongs to. Allows the supervisor to learn its // workspace context for subsequent workspace-scoped RPCs. string workspace = 10; + // Gateway-configured posture for rejected policy generations. Valid values + // are "fail_closed" and "retain_last_valid". Unknown or empty values must + // be treated as fail_closed by the supervisor. + string policy_validation_failure_mode = 11; } // Connection details for one operator-registered supervisor middleware service. From ce9bf500d3e4f7f04c9f068bb8f5143cc0db90ae Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Tue, 21 Jul 2026 11:26:42 -0700 Subject: [PATCH 13/30] test(network): name proxy contracts by behavior Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- crates/openshell-sandbox/src/lib.rs | 8 ++-- .../openshell-sandbox/src/metadata_server.rs | 4 +- .../openshell-supervisor-network/src/opa.rs | 2 +- .../openshell-supervisor-network/src/proxy.rs | 14 +++---- .../src/proxy/destination.rs | 2 +- .../tests/{phase0.rs => compatibility.rs} | 42 +++++++++---------- .../src/bypass_monitor/mod.rs | 2 +- 7 files changed, 37 insertions(+), 37 deletions(-) rename crates/openshell-supervisor-network/src/proxy/tests/{phase0.rs => compatibility.rs} (95%) diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 668d1c6c56..cdbf7d6894 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -4182,9 +4182,9 @@ filesystem_policy: assert_eq!(config["unmapped"]["previous_policy_active"], true); assert!( config["message"] - .as_str() - .unwrap() - .contains("previous policy IS active") - ); + .as_str() + .unwrap() + .contains("previous policy IS active") + ); } } diff --git a/crates/openshell-sandbox/src/metadata_server.rs b/crates/openshell-sandbox/src/metadata_server.rs index c618d8e771..cba614e496 100644 --- a/crates/openshell-sandbox/src/metadata_server.rs +++ b/crates/openshell-sandbox/src/metadata_server.rs @@ -175,7 +175,7 @@ mod tests { } #[tokio::test] - async fn phase0_metadata_loopback_dispatches_method_path_and_response() { + async fn metadata_loopback_dispatches_method_path_and_response() { let (requests_tx, mut requests_rx) = mpsc::unbounded_channel(); let handler = RecordingHandler { requests: requests_tx, @@ -202,7 +202,7 @@ mod tests { } #[tokio::test] - async fn phase0_metadata_loopback_rejects_oversized_headers_before_handler() { + async fn metadata_loopback_rejects_oversized_headers_before_handler() { let (requests_tx, mut requests_rx) = mpsc::unbounded_channel(); let handler = RecordingHandler { requests: requests_tx, diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index afc8b8e99c..d6af02a9f0 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -5283,7 +5283,7 @@ process: } #[test] - fn phase0_overlapping_policy_outputs_are_snapshotted_independently() { + fn overlapping_policy_outputs_are_snapshotted_independently() { let engine = OpaEngine::from_strings(TEST_POLICY, OVERLAPPING_L7_TEST_DATA) .expect("engine should load overlapping data"); let input = NetworkInput { diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 6452223eb7..b4d7ccbc33 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -5236,7 +5236,7 @@ network_policies: {} } #[test] - fn forward_policy_denial_ocsf_includes_validation_rationale() { + fn forward_policy_denial_ocsf_includes_validation_rationale() { let reason = "policy validation failed; fail-closed quarantine is active; candidate version 7 rejected: conflicting tls metadata"; let event = build_forward_policy_deny_ocsf_event( "127.0.0.1:45123".parse().unwrap(), @@ -5252,10 +5252,10 @@ network_policies: {} ); let json = event.to_json().unwrap(); - assert_eq!(json["status_detail"], reason); - assert_eq!(json["action"], "Denied"); - assert_eq!(json["disposition"], "Blocked"); - } + assert_eq!(json["status_detail"], reason); + assert_eq!(json["action"], "Denied"); + assert_eq!(json["disposition"], "Blocked"); + } #[test] fn endpoint_only_opa_allows_declared_endpoint_without_process_identity() { @@ -10396,6 +10396,6 @@ network_policies: assert_eq!(res, 3); assert_eq!(unk, 2); } - #[path = "phase0.rs"] - mod phase0; + #[path = "compatibility.rs"] + mod compatibility; } diff --git a/crates/openshell-supervisor-network/src/proxy/destination.rs b/crates/openshell-supervisor-network/src/proxy/destination.rs index c5cbc1776d..532e2e995f 100644 --- a/crates/openshell-supervisor-network/src/proxy/destination.rs +++ b/crates/openshell-supervisor-network/src/proxy/destination.rs @@ -250,7 +250,7 @@ mod tests { } #[test] - fn phase0_validation_mode_precedence_is_explicit_and_stable() { + fn validation_mode_precedence_is_explicit_and_stable() { let trusted_ip = IpAddr::V4(Ipv4Addr::new(169, 254, 1, 2)); let trusted = build_validation_plan( "host.openshell.internal", diff --git a/crates/openshell-supervisor-network/src/proxy/tests/phase0.rs b/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs similarity index 95% rename from crates/openshell-supervisor-network/src/proxy/tests/phase0.rs rename to crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs index 527c0b2dc1..d4097b5938 100644 --- a/crates/openshell-supervisor-network/src/proxy/tests/phase0.rs +++ b/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Compatibility fixtures for RFC 0005 Phase 0. +//! Compatibility and regression contracts for the shared proxy egress pipeline. use super::*; use std::io::Write; @@ -27,7 +27,7 @@ fn allowed_decision(intent: EgressIntent) -> EgressDecision { EgressDecision { intent, action: NetworkAction::Allow { - matched_policy: Some("phase0".to_string()), + matched_policy: Some("proxy_compatibility".to_string()), }, l4_policy_generation: 0, identity: ProcessIdentityEvidence::Available, @@ -98,7 +98,7 @@ async fn destination_denials_preserve_adapter_specific_wire_contracts() { for (kind, detail) in cases { let denial = DestinationDenial { kind, - reason: "phase0 destination failure".to_string(), + reason: "proxy compatibility destination failure".to_string(), }; let peer: SocketAddr = "127.0.0.1:41000".parse().unwrap(); @@ -142,7 +142,7 @@ async fn destination_denials_preserve_adapter_specific_wire_contracts() { "42", "/usr/bin/sh", "curl", - "phase0", + "proxy_compatibility", &allowed_decision(EgressIntent::forward_http( "target.example".to_string(), 8080, @@ -213,7 +213,7 @@ fn representative_adapter_denials_preserve_ocsf_fields() { "42", "/usr/bin/sh", "curl --proxy", - "phase0", + "proxy_compatibility", &allowed_decision(EgressIntent::forward_http( "target.example".to_string(), 8080, @@ -263,7 +263,7 @@ fn representative_adapter_denials_preserve_ocsf_fields() { assert_eq!(forward["dst_endpoint"]["domain"], "target.example"); assert_eq!(forward["dst_endpoint"]["port"], 8080); assert_eq!(forward["http_request"]["http_method"], "POST"); - assert_eq!(forward["firewall_rule"]["name"], "phase0"); + assert_eq!(forward["firewall_rule"]["name"], "proxy_compatibility"); assert_eq!(forward["firewall_rule"]["type"], "ssrf"); assert_eq!( forward["message"], @@ -283,7 +283,7 @@ fn representative_adapter_allows_preserve_ocsf_fields() { "42", "/usr/bin/sh", "curl --proxy", - "phase0", + "proxy_compatibility", true, )) .unwrap(); @@ -296,7 +296,7 @@ fn representative_adapter_allows_preserve_ocsf_fields() { assert_eq!(connect["dst_endpoint"]["domain"], "target.example"); assert_eq!(connect["dst_endpoint"]["port"], 8443); assert_eq!(connect["actor"]["process"]["name"], "/usr/bin/curl"); - assert_eq!(connect["firewall_rule"]["name"], "phase0"); + assert_eq!(connect["firewall_rule"]["name"], "proxy_compatibility"); assert_eq!(connect["firewall_rule"]["type"], "opa"); assert_eq!(connect["message"], "CONNECT_L7 allowed target.example:8443"); @@ -310,7 +310,7 @@ fn representative_adapter_allows_preserve_ocsf_fields() { "42", "/usr/bin/sh", "curl --proxy", - "phase0", + "proxy_compatibility", )) .unwrap(); assert_eq!(forward["class_name"], "HTTP Activity"); @@ -322,7 +322,7 @@ fn representative_adapter_allows_preserve_ocsf_fields() { assert_eq!(forward["dst_endpoint"]["domain"], "target.example"); assert_eq!(forward["dst_endpoint"]["port"], 8080); assert_eq!(forward["http_request"]["http_method"], "GET"); - assert_eq!(forward["firewall_rule"]["name"], "phase0"); + assert_eq!(forward["firewall_rule"]["name"], "proxy_compatibility"); assert_eq!(forward["firewall_rule"]["type"], "opa"); assert_eq!( forward["message"], @@ -335,8 +335,8 @@ fn poisoned_engine() -> OpaEngine { include_str!("../../../data/sandbox-policy.rego"), r#" network_policies: - phase0: - name: phase0 + proxy_compatibility: + name: proxy_compatibility endpoints: - host: target.example port: 443 @@ -397,8 +397,8 @@ fn identity_required_policy_accepts_real_binary_and_rejects_empty_exec_path() { include_str!("../../../data/sandbox-policy.rego"), r#" network_policies: - phase0: - name: phase0 + proxy_compatibility: + name: proxy_compatibility endpoints: - host: target.example port: 443 @@ -524,7 +524,7 @@ async fn exercise_benchmark_request(proxy_addr: SocketAddr, target: SocketAddr, format!("CONNECT {authority} HTTP/1.1\r\nHost: {authority}\r\n\r\n") } else { format!( - "GET http://{authority}/phase0 HTTP/1.1\r\nHost: {authority}\r\nConnection: close\r\n\r\n" + "GET http://{authority}/proxy-baseline HTTP/1.1\r\nHost: {authority}\r\nConnection: close\r\n\r\n" ) }; client.write_all(request.as_bytes()).await.unwrap(); @@ -534,10 +534,10 @@ async fn exercise_benchmark_request(proxy_addr: SocketAddr, target: SocketAddr, } /// Run with: -/// `cargo test -p openshell-supervisor-network phase0_proxy_performance_baseline -- --ignored --nocapture --test-threads=1` +/// `cargo test -p openshell-supervisor-network proxy_performance_baseline -- --ignored --nocapture --test-threads=1` #[test] -#[ignore = "manual Phase 0 allocation/query/latency baseline"] -fn phase0_proxy_performance_baseline() { +#[ignore = "manual proxy allocation/query/latency baseline"] +fn proxy_performance_baseline() { temp_env::with_vars( [( openshell_core::sandbox_env::NETWORK_BINARY_IDENTITY, @@ -559,8 +559,8 @@ fn phase0_proxy_performance_baseline() { let policy = format!( r#" network_policies: - phase0: - name: phase0 + proxy_compatibility: + name: proxy_compatibility endpoints: - host: {host} port: {port} @@ -642,7 +642,7 @@ network_policies: "{}", serde_json::json!({ "iterations": iterations, - "phase0_proxy_baseline": results, + "proxy_performance_baseline": results, "scenario": "declared_loopback_destination_denied", "schema_version": 1, }) diff --git a/crates/openshell-supervisor-process/src/bypass_monitor/mod.rs b/crates/openshell-supervisor-process/src/bypass_monitor/mod.rs index f04a2baefe..3dda53b2c3 100644 --- a/crates/openshell-supervisor-process/src/bypass_monitor/mod.rs +++ b/crates/openshell-supervisor-process/src/bypass_monitor/mod.rs @@ -496,7 +496,7 @@ mod tests { } #[test] - fn phase0_bypass_ocsf_contract_is_stable() { + fn bypass_ocsf_contract_is_stable() { let event = BypassEvent { dst_addr: "93.184.216.34".to_string(), dst_port: 443, From 867bd09b4577d0678da1a2521f58a9d0918fb8ed Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Thu, 23 Jul 2026 15:28:15 -0700 Subject: [PATCH 14/30] fix(network): align overlap validation with endpoint selection Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- architecture/security-policy.md | 4 +- crates/openshell-policy/src/ambiguity.rs | 24 +++++++- .../src/proxy/egress.rs | 1 + .../src/bypass_monitor/mod.rs | 9 ++- docs/sandboxes/policies.mdx | 2 +- e2e/rust/tests/proxy_egress_pipeline.rs | 61 +++++++++++++------ 6 files changed, 73 insertions(+), 28 deletions(-) diff --git a/architecture/security-policy.md b/architecture/security-policy.md index a765994602..4b9f3cebf2 100644 --- a/architecture/security-policy.md +++ b/architecture/security-policy.md @@ -102,7 +102,9 @@ The supervisor validates complete effective policy generations before activation. Overlapping endpoint selectors may contribute request allow and deny rules only when their connection and request-processing metadata agree; conflicting TLS, destination, credential, parser, or enforcement metadata -rejects the complete generation. +rejects the complete generation. Plain L4 endpoints do not contribute +request-processing metadata, so they may overlap an L7 endpoint when their +connection metadata agrees. The `[openshell.gateway] policy_validation_failure_mode` configuration controls rejected generations. It defaults to `fail_closed`, which publishes a quarantine diff --git a/crates/openshell-policy/src/ambiguity.rs b/crates/openshell-policy/src/ambiguity.rs index 16b4e3653a..5da1e0299f 100644 --- a/crates/openshell-policy/src/ambiguity.rs +++ b/crates/openshell-policy/src/ambiguity.rs @@ -89,7 +89,10 @@ pub fn find_endpoint_ambiguities(policy: &SandboxPolicy) -> Vec Vec< conflicts } +/// Keep request-pipeline ambiguity checks aligned with Rego's +/// `endpoint_has_extended_config` predicate. Plain L4 endpoints authorize a +/// destination but do not participate in endpoint-config selection, so they +/// cannot compete with the single L7/connection-config endpoint selected for +/// that request. +fn endpoint_contributes_request_pipeline_metadata(endpoint: &NetworkEndpoint) -> bool { + !endpoint.protocol.is_empty() || !endpoint.allowed_ips.is_empty() || !endpoint.tls.is_empty() +} + fn request_pipeline_conflicts(left: &NetworkEndpoint, right: &NetworkEndpoint) -> Vec { let mut conflicts = Vec::new(); push_conflict( @@ -507,6 +519,16 @@ mod tests { assert!(find_endpoint_ambiguities(&policy_with(left, right)).is_empty()); } + #[test] + fn plain_l4_endpoint_does_not_compete_with_l7_endpoint_metadata() { + let left = endpoint("api.example.com", 443); + let mut right = endpoint("api.example.com", 443); + right.protocol = "rest".to_string(); + right.enforcement = "enforce".to_string(); + + assert!(find_endpoint_ambiguities(&policy_with(left, right)).is_empty()); + } + #[test] fn disjoint_path_specific_protocols_may_overlap() { let mut left = endpoint("api.example.com", 443); diff --git a/crates/openshell-supervisor-network/src/proxy/egress.rs b/crates/openshell-supervisor-network/src/proxy/egress.rs index bf1b05131b..f059175cfa 100644 --- a/crates/openshell-supervisor-network/src/proxy/egress.rs +++ b/crates/openshell-supervisor-network/src/proxy/egress.rs @@ -91,6 +91,7 @@ impl EgressIntent { pub(super) enum IdentityUnavailableReason { EndpointOnlyMode, LookupFailed, + #[cfg(not(target_os = "linux"))] UnsupportedPlatform, } diff --git a/crates/openshell-supervisor-process/src/bypass_monitor/mod.rs b/crates/openshell-supervisor-process/src/bypass_monitor/mod.rs index 3dda53b2c3..44847b0d13 100644 --- a/crates/openshell-supervisor-process/src/bypass_monitor/mod.rs +++ b/crates/openshell-supervisor-process/src/bypass_monitor/mod.rs @@ -86,11 +86,10 @@ fn build_bypass_ocsf_events( let hint = hint_for_event(event); let reason = "direct connection bypassed HTTP CONNECT proxy"; let dst_port = event.dst_port.to_string(); - let dst_ep = if let Ok(ip) = event.dst_addr.parse::() { - Endpoint::from_ip(ip, event.dst_port) - } else { - Endpoint::from_domain(&event.dst_addr, event.dst_port) - }; + let dst_ep = event.dst_addr.parse::().map_or_else( + |_| Endpoint::from_domain(&event.dst_addr, event.dst_port), + |ip| Endpoint::from_ip(ip, event.dst_port), + ); let net_event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Refuse) diff --git a/docs/sandboxes/policies.mdx b/docs/sandboxes/policies.mdx index fce3841d2a..b66fcd56ca 100644 --- a/docs/sandboxes/policies.mdx +++ b/docs/sandboxes/policies.mdx @@ -203,7 +203,7 @@ The following steps outline the hot-reload policy update workflow. ### Validation failures -OpenShell validates a complete candidate policy before activating any part of it. Endpoints may overlap when their connection and request-processing metadata agree. For example, two `api.example.com:443` REST entries can contribute different allow and deny rules when they use the same TLS, destination, credential, parser, and enforcement settings. OpenShell rejects the candidate when overlapping exact or wildcard host selectors disagree on those fields. Path-scoped endpoints may use different protocols only when their path selectors cannot match the same request. +OpenShell validates a complete candidate policy before activating any part of it. Endpoints may overlap when their connection and request-processing metadata agree. For example, two `api.example.com:443` REST entries can contribute different allow and deny rules when they use the same TLS, destination, credential, parser, and enforcement settings. A plain L4 endpoint may overlap an L7 endpoint because it authorizes the destination without contributing request-processing metadata. OpenShell rejects the candidate when overlapping exact or wildcard host selectors can both contribute endpoint configuration and disagree on those fields. Path-scoped endpoints may use different protocols only when their path selectors cannot match the same request. The gateway's `policy_validation_failure_mode` configuration determines what happens after rejection. Set it under `[openshell.gateway]` in `gateway.toml`. Its default is `fail_closed`: diff --git a/e2e/rust/tests/proxy_egress_pipeline.rs b/e2e/rust/tests/proxy_egress_pipeline.rs index 28d2d6f1d5..683d8bcf60 100644 --- a/e2e/rust/tests/proxy_egress_pipeline.rs +++ b/e2e/rust/tests/proxy_egress_pipeline.rs @@ -64,6 +64,36 @@ async fn run_cli(args: &[&str]) -> Result { Ok(combined) } +async fn wait_for_sandbox_logs( + sandbox_name: &str, + expected: impl Fn(&str) -> bool, +) -> Result { + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(10); + + loop { + let logs = run_cli(&[ + "logs", + sandbox_name, + "-n", + "500", + "--since", + "2m", + "--source", + "sandbox", + ]) + .await?; + if expected(&logs) { + return Ok(logs); + } + if tokio::time::Instant::now() >= deadline { + return Err(format!( + "timed out waiting for expected sandbox logs:\n{logs}" + )); + } + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + } +} + async fn delete_provider(name: &str) { let mut cmd = openshell_cmd(); cmd.args(["provider", "delete", name]) @@ -1023,16 +1053,12 @@ async fn ambiguous_policy_update_fails_closed_without_contacting_upstream() { "quarantined requests must not contact the upstream server" ); - let logs = run_cli(&[ - "logs", - &guard.name, - "-n", - "500", - "--since", - "2m", - "--source", - "sandbox", - ]) + let logs = wait_for_sandbox_logs(&guard.name, |logs| { + logs.contains("configured_mode=fail_closed effective_mode=fail_closed") + && logs.contains("previous policy IS NOT active") + && logs.contains("conflicting metadata") + && logs.contains("tls") + }) .await .expect("fetch sandbox logs after rejection"); assert!( @@ -1455,16 +1481,11 @@ print("UNINSPECTABLE_MIDDLEWARE_BLOCKED") "uninspectable payload reached upstream before middleware denial" ); - let logs = run_cli(&[ - "logs", - &guard.name, - "-n", - "500", - "--since", - "2m", - "--source", - "sandbox", - ]) + let logs = wait_for_sandbox_logs(&guard.name, |logs| { + logs.contains("openshell.middleware.traffic_uninspectable") + && logs + .contains("Unsupported tunnel protocol cannot be inspected by required middleware") + }) .await .expect("fetch sandbox logs after middleware denial"); assert!( From fa0a80c6304870a746254a62e2e425be4582a17e Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Thu, 23 Jul 2026 15:38:45 -0700 Subject: [PATCH 15/30] test(network): expect hard loopback denial Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- crates/openshell-supervisor-network/src/proxy.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index b4d7ccbc33..27aa139ef0 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -9643,7 +9643,10 @@ network_policies: "internal forward destination must get the SSRF 403; got: {response:?}" ); assert!(response.contains("ssrf_denied")); - assert!(response.contains("GET 127.0.0.1:80 blocked: allowed_ips check failed")); + assert!( + response.contains("GET 127.0.0.1:80 blocked: internal address"), + "loopback must use the hard internal-address denial; got: {response:?}" + ); assert_eq!(denial_stages, ["ssrf"]); } From cbc3ab26bb54b6349f15f9f3e0686d4d5129ea6b Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Thu, 23 Jul 2026 15:46:49 -0700 Subject: [PATCH 16/30] test(network): match declared endpoint denial Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- crates/openshell-supervisor-network/src/proxy.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 27aa139ef0..69b2c44f9a 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -9644,8 +9644,8 @@ network_policies: ); assert!(response.contains("ssrf_denied")); assert!( - response.contains("GET 127.0.0.1:80 blocked: internal address"), - "loopback must use the hard internal-address denial; got: {response:?}" + response.contains("GET 127.0.0.1:80 blocked: declared endpoint check failed"), + "an explicit loopback endpoint must fail declared-endpoint validation; got: {response:?}" ); assert_eq!(denial_stages, ["ssrf"]); } From 39205ef6463130bf2ab8a1ada747ee9f75941d51 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Thu, 23 Jul 2026 16:12:10 -0700 Subject: [PATCH 17/30] fix(policy): preserve path-specific endpoint overrides Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- architecture/security-policy.md | 4 +++- crates/openshell-policy/src/ambiguity.rs | 29 ++++++++++++++++++++++++ docs/sandboxes/policies.mdx | 2 +- 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/architecture/security-policy.md b/architecture/security-policy.md index 4b9f3cebf2..eeccfc41ee 100644 --- a/architecture/security-policy.md +++ b/architecture/security-policy.md @@ -104,7 +104,9 @@ deny rules only when their connection and request-processing metadata agree; conflicting TLS, destination, credential, parser, or enforcement metadata rejects the complete generation. Plain L4 endpoints do not contribute request-processing metadata, so they may overlap an L7 endpoint when their -connection metadata agrees. +connection metadata agrees. When request paths overlap, a path endpoint with a +higher specificity rank deterministically overrides broader request-processing +metadata. Equally specific overlapping endpoints must agree. The `[openshell.gateway] policy_validation_failure_mode` configuration controls rejected generations. It defaults to `fail_closed`, which publishes a quarantine diff --git a/crates/openshell-policy/src/ambiguity.rs b/crates/openshell-policy/src/ambiguity.rs index 5da1e0299f..c84afc69e6 100644 --- a/crates/openshell-policy/src/ambiguity.rs +++ b/crates/openshell-policy/src/ambiguity.rs @@ -92,6 +92,8 @@ pub fn find_endpoint_ambiguities(policy: &SandboxPolicy) -> Vec bool { glob_patterns_overlap(left, right, '/') } +/// Match the runtime route-selection rank used by `L7EndpointConfig`. +/// +/// Overlapping endpoints with different ranks do not compete for request +/// metadata: the endpoint with the more-specific path wins. Equal-rank +/// overlaps must agree because iteration order would otherwise decide which +/// parser, credential handling, or enforcement behavior applies. +fn path_selector_specificity(path: &str) -> usize { + if path.is_empty() { + 0 + } else { + path.chars().filter(|character| *character != '*').count() + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum GlobToken { Literal(char), @@ -541,6 +557,19 @@ mod tests { assert!(find_endpoint_ambiguities(&policy_with(left, right)).is_empty()); } + #[test] + fn more_specific_path_may_override_request_pipeline_metadata() { + let mut left = endpoint("api.example.com", 443); + left.protocol = "rest".to_string(); + left.enforcement = "enforce".to_string(); + let mut right = endpoint("api.example.com", 443); + right.path = "/graphql".to_string(); + right.protocol = "graphql".to_string(); + right.enforcement = "enforce".to_string(); + + assert!(find_endpoint_ambiguities(&policy_with(left, right)).is_empty()); + } + #[test] fn exact_wildcard_tls_conflict_is_rejected() { let mut left = endpoint("*.example.com", 443); diff --git a/docs/sandboxes/policies.mdx b/docs/sandboxes/policies.mdx index b66fcd56ca..2fae38f909 100644 --- a/docs/sandboxes/policies.mdx +++ b/docs/sandboxes/policies.mdx @@ -203,7 +203,7 @@ The following steps outline the hot-reload policy update workflow. ### Validation failures -OpenShell validates a complete candidate policy before activating any part of it. Endpoints may overlap when their connection and request-processing metadata agree. For example, two `api.example.com:443` REST entries can contribute different allow and deny rules when they use the same TLS, destination, credential, parser, and enforcement settings. A plain L4 endpoint may overlap an L7 endpoint because it authorizes the destination without contributing request-processing metadata. OpenShell rejects the candidate when overlapping exact or wildcard host selectors can both contribute endpoint configuration and disagree on those fields. Path-scoped endpoints may use different protocols only when their path selectors cannot match the same request. +OpenShell validates a complete candidate policy before activating any part of it. Endpoints may overlap when their connection and request-processing metadata agree. For example, two `api.example.com:443` REST entries can contribute different allow and deny rules when they use the same TLS, destination, credential, parser, and enforcement settings. A plain L4 endpoint may overlap an L7 endpoint because it authorizes the destination without contributing request-processing metadata. A more-specific path endpoint may override request-processing metadata from a broader endpoint, such as a `/graphql` GraphQL endpoint alongside a general REST endpoint for the same host. OpenShell rejects the candidate when overlapping exact or wildcard host selectors can both contribute equally specific endpoint configuration and disagree on those fields. The gateway's `policy_validation_failure_mode` configuration determines what happens after rejection. Set it under `[openshell.gateway]` in `gateway.toml`. Its default is `fail_closed`: From b88c2450992f0e6685a18349b86dfae71fa057ed Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Thu, 23 Jul 2026 16:38:56 -0700 Subject: [PATCH 18/30] test(network): respect hard-blocked host gateways Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- e2e/rust/tests/proxy_egress_pipeline.rs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/e2e/rust/tests/proxy_egress_pipeline.rs b/e2e/rust/tests/proxy_egress_pipeline.rs index 683d8bcf60..35049bc74b 100644 --- a/e2e/rust/tests/proxy_egress_pipeline.rs +++ b/e2e/rust/tests/proxy_egress_pipeline.rs @@ -1224,10 +1224,21 @@ async fn explicit_allowed_ips_and_implicit_ip_literals_succeed_through_both_adap .lines() .find_map(|line| line.trim().strip_prefix("GATEWAY_IP=")) .expect("sandbox gateway IPv4 output") - .to_string(); - gateway_ip .parse::() - .expect("host gateway must resolve to IPv4 for this Docker e2e"); + .expect("host gateway must resolve to IPv4 for this e2e"); + + // Rootless Podman with pasta exposes its trusted host-gateway alias as a + // link-local address. The hostname receives a narrow runtime exemption, + // but the equivalent raw IP literal must remain hard-blocked. Other + // drivers still exercise the successful IP-literal path below. + if gateway_ip.is_loopback() || gateway_ip.is_link_local() || gateway_ip.is_unspecified() { + eprintln!( + "skipping IP-literal success assertions: host gateway {gateway_ip} is always blocked" + ); + return; + } + + let gateway_ip = gateway_ip.to_string(); let explicit_server = KeepAliveHttpServer::start() .await From 57b014ac2e3b3ca6e335782fb2cd4b90927de58a Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Mon, 27 Jul 2026 08:09:32 -0700 Subject: [PATCH 19/30] fix(network): reconcile proxy refactor with main Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- .../openshell-supervisor-network/src/proxy.rs | 17 +++++++---------- .../src/proxy/relay.rs | 1 + .../src/proxy/tests/compatibility.rs | 7 ++++++- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 69b2c44f9a..a5c66c1574 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -1289,7 +1289,7 @@ async fn handle_tcp_connection( deny_connect_destination( &mut client, &denial, - peer_addr, + workload_addr, &host_lc, port, &binary_str, @@ -1383,14 +1383,9 @@ async fn handle_tcp_connection( return Ok(()); } - let mut upstream = dial_upstream( - &upstream_proxy, - &host_lc, - port, - connector.addrs(), - ) - .await - .into_diagnostic()?; + let mut upstream = dial_upstream(&upstream_proxy, &host_lc, port, connector.addrs()) + .await + .into_diagnostic()?; debug!( "handle_tcp_connection dns_resolve_and_tcp_connect: {}ms host={host_lc}", @@ -4516,7 +4511,7 @@ async fn handle_forward_proxy( deny_forward_destination( client, &denial, - peer_addr, + workload_addr, method, &host_lc, port, @@ -9536,6 +9531,8 @@ network_policies: None, None, None, + AgentProposals::default(), + Arc::new(None), Arc::new(None), None, None, diff --git a/crates/openshell-supervisor-network/src/proxy/relay.rs b/crates/openshell-supervisor-network/src/proxy/relay.rs index cdc49f95bc..c16ea4a0ba 100644 --- a/crates/openshell-supervisor-network/src/proxy/relay.rs +++ b/crates/openshell-supervisor-network/src/proxy/relay.rs @@ -311,6 +311,7 @@ mod tests { activity_tx: None, dynamic_credentials: None, token_grant_resolver: None, + agent_proposals: openshell_core::proposals::AgentProposals::default(), } } diff --git a/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs b/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs index d4097b5938..1ffe08d884 100644 --- a/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs +++ b/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs @@ -439,7 +439,10 @@ fn identity_required_mode_is_explicitly_unsupported_off_linux() { ) .unwrap(); let decision = authorize_egress_intent( - "127.0.0.1:41000".parse().unwrap(), + crate::procfs::WorkloadProxyTcpConnection::new( + "127.0.0.1:41000".parse().unwrap(), + "127.0.0.1:3000".parse().unwrap(), + ), &engine, &BinaryIdentityCache::new(), &AtomicU32::new(1), @@ -594,6 +597,8 @@ network_policies: None, None, None, + AgentProposals::default(), + Arc::new(None), Arc::new(None), None, None, From 0cf33407eef6379169aef633a6a541ed515342b8 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Mon, 27 Jul 2026 11:45:10 -0700 Subject: [PATCH 20/30] fix(network): preserve CONNECT policy generation Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- .../openshell-supervisor-network/src/proxy.rs | 89 ++++++++++++- .../src/proxy/relay.rs | 126 ++++++++++++++---- 2 files changed, 181 insertions(+), 34 deletions(-) diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index a5c66c1574..bc093e439b 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -1272,6 +1272,22 @@ async fn handle_tcp_connection( return Ok(()); } + let connect_generation_guard = + match relay::pin_policy_generation(&opa_engine, decision.l4_policy_generation) { + Ok(guard) => guard, + Err(error) => { + reject_stale_connect_policy( + &mut client, + &host_lc, + port, + activity_tx.as_ref(), + error, + ) + .await?; + return Ok(()); + } + }; + // Resolve the route's TLS treatment up front. `query_tls_mode` reads only // the policy decision + host/port (no peeked bytes), so it is valid before // the `200`. The fail-closed refusal that consumes it runs after the SSRF/ @@ -1383,9 +1399,45 @@ async fn handle_tcp_connection( return Ok(()); } - let mut upstream = dial_upstream(&upstream_proxy, &host_lc, port, connector.addrs()) - .await - .into_diagnostic()?; + // CONNECT must use one policy generation from authorization through route + // hydration and relay startup. A later L7 lookup must never make a stale + // L4 allow appear current. + hydrate_l7_route(&opa_engine, &mut decision); + let l7_route = decision.endpoint.l7_route.as_ref(); + if let Err(error) = + relay::validate_route_generation(l7_route, connect_generation_guard.captured_generation()) + { + reject_stale_connect_policy(&mut client, &host_lc, port, activity_tx.as_ref(), error) + .await?; + return Ok(()); + } + + let upstream_result = tokio::select! { + result = dial_upstream(&upstream_proxy, &host_lc, port, connector.addrs()) => Some(result), + () = connect_generation_guard.wait_until_stale() => None, + }; + let Some(upstream_result) = upstream_result else { + reject_stale_connect_policy( + &mut client, + &host_lc, + port, + activity_tx.as_ref(), + miette::miette!( + "policy changed while CONNECT was dialing upstream \ + [captured_generation:{} current_generation:{}]", + connect_generation_guard.captured_generation(), + connect_generation_guard.current_generation(), + ), + ) + .await?; + return Ok(()); + }; + let mut upstream = upstream_result.into_diagnostic()?; + if let Err(error) = connect_generation_guard.ensure_current() { + reject_stale_connect_policy(&mut client, &host_lc, port, activity_tx.as_ref(), error) + .await?; + return Ok(()); + } debug!( "handle_tcp_connection dns_resolve_and_tcp_connect: {}ms host={host_lc}", @@ -1394,10 +1446,6 @@ async fn handle_tcp_connection( respond(&mut client, b"HTTP/1.1 200 Connection Established\r\n\r\n").await?; - // Check if endpoint has L7 config for protocol-aware inspection, and - // retain the generation for HTTP passthrough keep-alive tunnels. - hydrate_l7_route(&opa_engine, &mut decision); - let l7_route = decision.endpoint.l7_route.as_ref(); let should_inspect_l7 = l7_inspection_active(l7_route); // Log the allowed CONNECT — use CONNECT_L7 when L7 inspection follows, @@ -2523,6 +2571,33 @@ fn emit_l7_tunnel_close_after_policy_change(host: &str, port: u16, error: miette ocsf_emit!(event); } +async fn reject_stale_connect_policy( + client: &mut TcpStream, + host: &str, + port: u16, + activity_tx: Option<&ActivitySender>, + error: miette::Report, +) -> Result<()> { + warn!( + host, + port, + error = %error, + "CONNECT rejected because policy changed after L4 authorization" + ); + emit_l7_tunnel_close_after_policy_change(host, port, error); + emit_activity_simple(activity_tx, true, "policy_stale"); + respond( + client, + &build_json_error_response( + 403, + "Forbidden", + "policy_denied", + &format!("CONNECT {host}:{port} not permitted because policy changed"), + ), + ) + .await +} + /// Query L7 endpoint config from the OPA engine for an allowed egress decision. /// /// Returns `Some(L7EndpointConfig)` if the matched endpoint has L7 config (protocol field), diff --git a/crates/openshell-supervisor-network/src/proxy/relay.rs b/crates/openshell-supervisor-network/src/proxy/relay.rs index c16ea4a0ba..5eada877a1 100644 --- a/crates/openshell-supervisor-network/src/proxy/relay.rs +++ b/crates/openshell-supervisor-network/src/proxy/relay.rs @@ -95,6 +95,23 @@ pub(super) fn pin_l7_evaluator( opa_engine.clone_engine_for_tunnel(expected_generation) } +pub(super) fn validate_route_generation( + route: Option<&L7RouteSnapshot>, + expected_generation: u64, +) -> Result<()> { + if let Some(route) = route + && route.l7_policy_generation != expected_generation + { + return Err(miette::miette!( + "policy changed before CONNECT route hydration \ + [l4_generation:{} l7_generation:{}]", + expected_generation, + route.l7_policy_generation, + )); + } + Ok(()) +} + /// Prepare a generation-pinned HTTP relay at the adapter boundary. /// /// A stale generation preserves the established CONNECT behavior: emit the @@ -106,8 +123,17 @@ pub(super) fn prepare_http_relay<'a>( decision: &EgressDecision, request: &'a L7EvalContext, ) -> Option> { + if let Err(error) = validate_route_generation(route, decision.l4_policy_generation) { + emit_l7_tunnel_close_after_policy_change( + &decision.intent.destination.host, + decision.intent.destination.port, + error, + ); + return None; + } + let policy = if let Some(route) = route.filter(|route| !route.configs.is_empty()) { - let evaluator = match pin_l7_evaluator(opa_engine, route.l7_policy_generation) { + let evaluator = match pin_l7_evaluator(opa_engine, decision.l4_policy_generation) { Ok(evaluator) => evaluator, Err(error) => { emit_l7_tunnel_close_after_policy_change( @@ -128,20 +154,18 @@ pub(super) fn prepare_http_relay<'a>( evaluator: Box::new(evaluator), } } else { - let expected_generation = route.map_or(decision.l4_policy_generation, |route| { - route.l7_policy_generation - }); - let generation_guard = match pin_policy_generation(opa_engine, expected_generation) { - Ok(guard) => guard, - Err(error) => { - emit_l7_tunnel_close_after_policy_change( - &decision.intent.destination.host, - decision.intent.destination.port, - error, - ); - return None; - } - }; + let generation_guard = + match pin_policy_generation(opa_engine, decision.l4_policy_generation) { + Ok(guard) => guard, + Err(error) => { + emit_l7_tunnel_close_after_policy_change( + &decision.intent.destination.host, + decision.intent.destination.port, + error, + ); + return None; + } + }; PreparedHttpPolicy::Passthrough { generation_guard } }; @@ -160,10 +184,16 @@ pub(super) fn prepare_raw_relay( opa_engine: &OpaEngine, decision: &EgressDecision, ) -> Option { - let expected_generation = route.map_or(decision.l4_policy_generation, |route| { - route.l7_policy_generation - }); - match pin_policy_generation(opa_engine, expected_generation) { + if let Err(error) = validate_route_generation(route, decision.l4_policy_generation) { + emit_l7_tunnel_close_after_policy_change( + &decision.intent.destination.host, + decision.intent.destination.port, + error, + ); + return None; + } + + match pin_policy_generation(opa_engine, decision.l4_policy_generation) { Ok(guard) => Some(guard), Err(error) => { emit_l7_tunnel_close_after_policy_change( @@ -334,7 +364,7 @@ mod tests { } #[test] - fn empty_hydrated_route_pins_l7_lookup_generation() { + fn empty_hydrated_route_cannot_replace_stale_l4_generation() { let engine = OpaEngine::from_strings(POLICY_REGO, EMPTY_POLICY_DATA).unwrap(); let decision = decision(u64::MAX); let route = L7RouteSnapshot { @@ -343,15 +373,57 @@ mod tests { }; let request = request_context(); - let context = prepare_http_relay(Some(&route), &engine, &decision, &request) - .expect("the hydrated route generation should take precedence over stale L4 data"); - let PreparedHttpPolicy::Passthrough { generation_guard } = context.policy else { - panic!("empty L7 route should use a generation guard"); + assert!( + prepare_http_relay(Some(&route), &engine, &decision, &request).is_none(), + "a current L7 lookup must not freshen a stale L4 allow" + ); + } + + #[test] + fn inspected_route_cannot_replace_stale_l4_generation() { + let engine = OpaEngine::from_strings(POLICY_REGO, EMPTY_POLICY_DATA).unwrap(); + let decision = decision(u64::MAX); + let route = L7RouteSnapshot { + configs: vec![super::super::L7ConfigSnapshot { + config: crate::l7::L7EndpointConfig { + protocol: crate::l7::L7Protocol::Rest, + path: "/**".to_string(), + tls: crate::l7::TlsMode::Auto, + enforcement: crate::l7::EnforcementMode::Enforce, + graphql_max_body_bytes: crate::l7::graphql::DEFAULT_MAX_BODY_BYTES, + json_rpc_max_body_bytes: crate::l7::jsonrpc::DEFAULT_MAX_BODY_BYTES, + mcp_strict_tool_names: true, + allow_encoded_slash: false, + websocket_credential_rewrite: false, + request_body_credential_rewrite: false, + websocket_graphql_policy: false, + credential_signing: crate::l7::CredentialSigning::None, + signing_service: String::new(), + signing_region: String::new(), + }, + }], + l7_policy_generation: engine.current_generation(), }; + let request = request_context(); - assert_eq!( - generation_guard.captured_generation(), - route.l7_policy_generation + assert!( + prepare_http_relay(Some(&route), &engine, &decision, &request).is_none(), + "an inspected route must use the generation that authorized CONNECT" + ); + } + + #[test] + fn raw_route_cannot_replace_stale_l4_generation() { + let engine = OpaEngine::from_strings(POLICY_REGO, EMPTY_POLICY_DATA).unwrap(); + let decision = decision(u64::MAX); + let route = L7RouteSnapshot { + configs: vec![], + l7_policy_generation: engine.current_generation(), + }; + + assert!( + prepare_raw_relay(Some(&route), &engine, &decision).is_none(), + "a raw relay must not freshen a stale L4 allow" ); } From 0b115ed4fe816548c419afe879a32d3bffe81139 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Mon, 27 Jul 2026 11:46:30 -0700 Subject: [PATCH 21/30] fix(policy): cover runtime endpoint glob semantics Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- crates/openshell-policy/src/ambiguity.rs | 356 +++++++++++++++++++++-- 1 file changed, 335 insertions(+), 21 deletions(-) diff --git a/crates/openshell-policy/src/ambiguity.rs b/crates/openshell-policy/src/ambiguity.rs index c84afc69e6..661948b279 100644 --- a/crates/openshell-policy/src/ambiguity.rs +++ b/crates/openshell-policy/src/ambiguity.rs @@ -225,6 +225,16 @@ fn request_pipeline_conflicts(left: &NetworkEndpoint, right: &NetworkEndpoint) - &left.request_body_credential_rewrite, &right.request_body_credential_rewrite, ); + if left.protocol.eq_ignore_ascii_case("websocket") + && right.protocol.eq_ignore_ascii_case("websocket") + { + push_conflict( + &mut conflicts, + "websocket_graphql_policy", + &websocket_graphql_policy(left), + &websocket_graphql_policy(right), + ); + } push_conflict( &mut conflicts, "credential_signing", @@ -273,6 +283,26 @@ fn request_pipeline_conflicts(left: &NetworkEndpoint, right: &NetworkEndpoint) - conflicts } +fn websocket_graphql_policy(endpoint: &NetworkEndpoint) -> bool { + let allow_rule_has_graphql_fields = endpoint.rules.iter().any(|rule| { + rule.allow.as_ref().is_some_and(|allow| { + !allow.operation_type.is_empty() + || !allow.operation_name.is_empty() + || !allow.fields.is_empty() + }) + }); + let deny_rule_has_graphql_fields = endpoint.deny_rules.iter().any(|deny| { + !deny.operation_type.is_empty() + || !deny.operation_name.is_empty() + || !deny.fields.is_empty() + }); + + !endpoint.graphql_persisted_queries.is_empty() + || (!endpoint.persisted_queries.is_empty() && endpoint.persisted_queries != "deny") + || allow_rule_has_graphql_fields + || deny_rule_has_graphql_fields +} + fn push_conflict( conflicts: &mut Vec, field: &str, @@ -346,7 +376,7 @@ fn path_patterns_overlap(left: &str, right: &str) -> bool { { return true; } - glob_patterns_overlap(left, right, '/') + runtime_path_patterns_overlap(left, right) } /// Match the runtime route-selection rank used by `L7EndpointConfig`. @@ -364,12 +394,25 @@ fn path_selector_specificity(path: &str) -> usize { } #[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct CharacterRange { + start: char, + end: char, +} + +#[derive(Debug, Clone, PartialEq, Eq)] enum GlobToken { Literal(char), - Star { crosses_delimiter: bool }, + AnyChar, + CharacterClass { + ranges: Vec, + negated: bool, + }, + Star { + crosses_delimiter: bool, + }, } -fn tokenize_glob(pattern: &str) -> Vec { +fn tokenize_delimited_glob(pattern: &str) -> Vec { let chars = pattern.chars().collect::>(); let mut tokens = Vec::new(); let mut index = 0; @@ -391,14 +434,111 @@ fn tokenize_glob(pattern: &str) -> Vec { tokens } +fn tokenize_runtime_path_glob(pattern: &str) -> Option> { + let chars = pattern.chars().collect::>(); + let mut tokens = Vec::new(); + let mut index = 0; + while index < chars.len() { + match chars[index] { + '?' => { + tokens.push(GlobToken::AnyChar); + index += 1; + } + '*' => { + let start = index; + while index < chars.len() && chars[index] == '*' { + index += 1; + } + let count = index - start; + if count > 2 { + return None; + } + if count == 2 { + let starts_component = start == 0 || chars[start - 1] == '/'; + let ends_component = index == chars.len() || chars[index] == '/'; + if !starts_component || !ends_component { + return None; + } + if index < chars.len() && chars[index] == '/' { + index += 1; + } + } + // `glob::Pattern::matches` uses `require_literal_separator: + // false`, so both `*` and `**` can consume `/`. + tokens.push(GlobToken::Star { + crosses_delimiter: true, + }); + } + '[' => { + let negated = chars.get(index + 1) == Some(&'!'); + let content_start = index + if negated { 2 } else { 1 }; + let close = chars[content_start..] + .iter() + .position(|character| *character == ']') + .map(|offset| content_start + offset)?; + if close == content_start { + return None; + } + tokens.push(GlobToken::CharacterClass { + ranges: parse_character_ranges(&chars[content_start..close]), + negated, + }); + index = close + 1; + } + literal => { + tokens.push(GlobToken::Literal(literal)); + index += 1; + } + } + } + Some(tokens) +} + +fn parse_character_ranges(characters: &[char]) -> Vec { + let mut ranges = Vec::new(); + let mut index = 0; + while index < characters.len() { + if index + 2 < characters.len() && characters[index + 1] == '-' { + ranges.push(CharacterRange { + start: characters[index], + end: characters[index + 2], + }); + index += 3; + } else { + ranges.push(CharacterRange { + start: characters[index], + end: characters[index], + }); + index += 1; + } + } + ranges +} + +fn runtime_path_patterns_overlap(left: &str, right: &str) -> bool { + let (Some(left), Some(right)) = ( + tokenize_runtime_path_glob(left), + tokenize_runtime_path_glob(right), + ) else { + // Invalid path globs are rejected by ordinary policy validation. Keep + // ambiguity validation conservative if it is called independently. + return true; + }; + token_languages_overlap(&left, &right, '/') +} + /// Decide whether two delimiter-aware glob languages intersect. /// /// This is a small product-NFA search. `*` consumes any character except the /// delimiter and `**` consumes any character, including the delimiter. Star /// epsilon transitions and self-loops make the state space finite. fn glob_patterns_overlap(left: &str, right: &str, delimiter: char) -> bool { - let left = tokenize_glob(left); - let right = tokenize_glob(right); + let left = tokenize_delimited_glob(left); + let right = tokenize_delimited_glob(right); + token_languages_overlap(&left, &right, delimiter) +} + +fn token_languages_overlap(left: &[GlobToken], right: &[GlobToken], delimiter: char) -> bool { let mut queue = VecDeque::from([(0_usize, 0_usize)]); let mut seen = HashSet::new(); @@ -423,7 +563,7 @@ fn glob_patterns_overlap(left: &str, right: &str, delimiter: char) -> bool { let Some(right_token) = right.get(right_index) else { continue; }; - if tokens_share_character(*left_token, *right_token, delimiter) { + if tokens_share_character(left_token, right_token, delimiter) { let next_left = if matches!(left_token, GlobToken::Star { .. }) { left_index } else { @@ -440,28 +580,105 @@ fn glob_patterns_overlap(left: &str, right: &str, delimiter: char) -> bool { false } -fn tokens_share_character(left: GlobToken, right: GlobToken, delimiter: char) -> bool { - match (left, right) { - (GlobToken::Literal(left), GlobToken::Literal(right)) => left == right, - (GlobToken::Literal(value), GlobToken::Star { crosses_delimiter }) - | (GlobToken::Star { crosses_delimiter }, GlobToken::Literal(value)) => { - crosses_delimiter || value != delimiter +fn tokens_share_character(left: &GlobToken, right: &GlobToken, delimiter: char) -> bool { + let left_ranges = token_character_ranges(left, delimiter); + let right_ranges = token_character_ranges(right, delimiter); + left_ranges.iter().any(|left| { + right_ranges + .iter() + .any(|right| left.0 <= right.1 && right.0 <= left.1) + }) +} + +fn token_character_ranges(token: &GlobToken, delimiter: char) -> Vec<(u32, u32)> { + match token { + GlobToken::Literal(value) => vec![(u32::from(*value), u32::from(*value))], + GlobToken::AnyChar + | GlobToken::Star { + crosses_delimiter: true, + } => unicode_scalar_ranges(), + GlobToken::Star { + crosses_delimiter: false, + } => complement_ranges(&[(u32::from(delimiter), u32::from(delimiter))]), + GlobToken::CharacterClass { ranges, negated } => { + let ranges = normalize_ranges( + ranges + .iter() + .filter(|range| range.start <= range.end) + .map(|range| (u32::from(range.start), u32::from(range.end))) + .collect(), + ); + if *negated { + complement_ranges(&ranges) + } else { + ranges + } + } + } +} + +fn unicode_scalar_ranges() -> Vec<(u32, u32)> { + vec![(0, 0xD7FF), (0xE000, 0x0010_FFFF)] +} + +fn normalize_ranges(mut ranges: Vec<(u32, u32)>) -> Vec<(u32, u32)> { + ranges.sort_unstable(); + let mut normalized: Vec<(u32, u32)> = Vec::new(); + for (start, end) in ranges { + for (start, end) in intersect_with_unicode_scalars(start, end) { + if let Some(last) = normalized.last_mut() + && start <= last.1.saturating_add(1) + { + last.1 = last.1.max(end); + } else { + normalized.push((start, end)); + } + } + } + normalized +} + +fn intersect_with_unicode_scalars(start: u32, end: u32) -> Vec<(u32, u32)> { + unicode_scalar_ranges() + .into_iter() + .filter_map(|(scalar_start, scalar_end)| { + let start = start.max(scalar_start); + let end = end.min(scalar_end); + (start <= end).then_some((start, end)) + }) + .collect() +} + +fn complement_ranges(ranges: &[(u32, u32)]) -> Vec<(u32, u32)> { + let ranges = normalize_ranges(ranges.to_vec()); + let mut complement = Vec::new(); + for (universe_start, universe_end) in unicode_scalar_ranges() { + let mut cursor = universe_start; + for &(start, end) in &ranges { + if end < universe_start || start > universe_end { + continue; + } + let start = start.max(universe_start); + let end = end.min(universe_end); + if cursor < start { + complement.push((cursor, start - 1)); + } + cursor = end.saturating_add(1); + if cursor > universe_end { + break; + } + } + if cursor <= universe_end { + complement.push((cursor, universe_end)); } - ( - GlobToken::Star { - crosses_delimiter: _, - }, - GlobToken::Star { - crosses_delimiter: _, - }, - ) => true, } + complement } #[cfg(test)] mod tests { use super::*; - use openshell_core::proto::{NetworkBinary, NetworkPolicyRule}; + use openshell_core::proto::{L7Allow, L7Rule, NetworkBinary, NetworkPolicyRule}; fn endpoint(host: &str, port: u32) -> NetworkEndpoint { NetworkEndpoint { @@ -557,6 +774,58 @@ mod tests { assert!(find_endpoint_ambiguities(&policy_with(left, right)).is_empty()); } + #[test] + fn question_mark_path_overlap_is_detected() { + let mut left = endpoint("api.example.com", 443); + left.path = "/v?".to_string(); + left.protocol = "rest".to_string(); + let mut right = endpoint("api.example.com", 443); + right.path = "/v1".to_string(); + right.protocol = "graphql".to_string(); + + let ambiguities = find_endpoint_ambiguities(&policy_with(left, right)); + assert_eq!(ambiguities.len(), 1); + assert!( + ambiguities[0] + .conflicts + .iter() + .any(|field| field.contains("protocol")) + ); + } + + #[test] + fn overlapping_character_class_paths_are_detected() { + let mut left = endpoint("api.example.com", 443); + left.path = "/v[12]".to_string(); + left.protocol = "rest".to_string(); + let mut right = endpoint("api.example.com", 443); + right.path = "/v[23]".to_string(); + right.protocol = "graphql".to_string(); + + assert_eq!( + find_endpoint_ambiguities(&policy_with(left, right)).len(), + 1 + ); + } + + #[test] + fn disjoint_character_class_paths_may_overlap_by_host_and_port() { + let mut left = endpoint("api.example.com", 443); + left.path = "/v[12]".to_string(); + left.protocol = "rest".to_string(); + let mut right = endpoint("api.example.com", 443); + right.path = "/v[34]".to_string(); + right.protocol = "graphql".to_string(); + + assert!(find_endpoint_ambiguities(&policy_with(left, right)).is_empty()); + } + + #[test] + fn negated_character_class_paths_follow_runtime_glob_semantics() { + assert!(runtime_path_patterns_overlap("/v[!0]", "/v1")); + assert!(!runtime_path_patterns_overlap("/v[!0]", "/v0")); + } + #[test] fn more_specific_path_may_override_request_pipeline_metadata() { let mut left = endpoint("api.example.com", 443); @@ -627,6 +896,51 @@ mod tests { ); } + #[test] + fn websocket_graphql_classification_conflict_is_rejected() { + let mut graphql = endpoint("api.example.com", 443); + graphql.protocol = "websocket".to_string(); + graphql.rules.push(L7Rule { + allow: Some(L7Allow { + operation_type: "subscription".to_string(), + ..Default::default() + }), + }); + let mut transport = endpoint("api.example.com", 443); + transport.protocol = "websocket".to_string(); + transport.rules.push(L7Rule { + allow: Some(L7Allow { + method: "WEBSOCKET_TEXT".to_string(), + ..Default::default() + }), + }); + + let ambiguities = find_endpoint_ambiguities(&policy_with(graphql, transport)); + assert_eq!(ambiguities.len(), 1); + assert!( + ambiguities[0] + .conflicts + .iter() + .any(|field| field.contains("websocket_graphql_policy")) + ); + } + + #[test] + fn matching_websocket_graphql_classification_is_compatible() { + let mut left = endpoint("api.example.com", 443); + left.protocol = "websocket".to_string(); + left.persisted_queries = "allow_registered".to_string(); + let mut right = left.clone(); + right.rules.push(L7Rule { + allow: Some(L7Allow { + operation_name: "Events".to_string(), + ..Default::default() + }), + }); + + assert!(find_endpoint_ambiguities(&policy_with(left, right)).is_empty()); + } + #[test] fn different_binary_lists_do_not_hide_endpoint_ambiguity() { let mut left = endpoint("api.example.com", 443); From 544635dc1eeb12bdf69cdfbc983792a21ee6433d Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Mon, 27 Jul 2026 11:48:26 -0700 Subject: [PATCH 22/30] fix(server): reject ambiguous policies before persistence Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- crates/openshell-server/src/grpc/policy.rs | 431 ++++++++++++++++++- crates/openshell-server/src/grpc/provider.rs | 200 ++++++++- crates/openshell-server/src/grpc/sandbox.rs | 7 + e2e/rust/tests/proxy_egress_pipeline.rs | 79 +--- 4 files changed, 635 insertions(+), 82 deletions(-) diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index 05ac8d5feb..b345a8210b 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -987,9 +987,27 @@ async fn auto_approve_chunk( return Ok(()); } - let (version, hash) = - merge_chunk_into_policy(state.store.as_ref(), sandbox_id, context.workspace, &chunk) - .await?; + let provider_names = context + .sandbox + .spec + .as_ref() + .map(|spec| spec.providers.as_slice()) + .unwrap_or_default(); + let provider_layers = provider_policy_layers_for_sandbox( + state, + context.workspace, + context.sandbox, + provider_names, + ) + .await?; + let (version, hash) = merge_chunk_into_policy( + state.store.as_ref(), + sandbox_id, + context.workspace, + &chunk, + &provider_layers, + ) + .await?; let chunk_summary = summarize_draft_chunk_rule(&chunk)?; let now_ms = current_time_ms(); @@ -1091,6 +1109,103 @@ async fn current_effective_policy_for_sandbox( Ok(policy) } +fn validate_endpoint_ambiguities(policy: &ProtoSandboxPolicy) -> Result<(), Status> { + let ambiguities = openshell_policy::find_endpoint_ambiguities(policy); + if ambiguities.is_empty() { + return Ok(()); + } + Err(Status::failed_precondition(format!( + "network endpoint ambiguity validation failed:\n{}", + ambiguities + .iter() + .map(ToString::to_string) + .collect::>() + .join("\n") + ))) +} + +pub(super) fn validate_candidate_effective_policy( + base_policy: &ProtoSandboxPolicy, + provider_layers: &[ProviderPolicyLayer], +) -> Result<(), Status> { + let effective_policy = if provider_layers.is_empty() { + base_policy.clone() + } else { + compose_effective_policy(base_policy, provider_layers) + }; + validate_endpoint_ambiguities(&effective_policy) +} + +async fn provider_policy_layers_for_sandbox( + state: &ServerState, + workspace: &str, + sandbox: &Sandbox, + provider_names: &[String], +) -> Result, Status> { + let global_settings = load_global_settings(state.store.as_ref()).await?; + if decode_policy_from_global_settings(&global_settings)?.is_some() + || !bool_setting_enabled(&global_settings, settings::PROVIDERS_V2_ENABLED_KEY)? + { + return Ok(Vec::new()); + } + let catalog = state + .provider_profile_sources + .snapshot_catalog(state.store.as_ref(), workspace) + .await?; + let layers = profile_provider_policy_layers_with_catalog( + state.store.as_ref(), + &catalog, + workspace, + provider_names, + ) + .await?; + debug!( + sandbox_id = %sandbox.object_id(), + provider_layer_count = layers.len(), + "Composed candidate provider policy layers for ambiguity validation" + ); + Ok(layers) +} + +pub(super) async fn current_base_policy_for_sandbox( + store: &Store, + sandbox: &Sandbox, +) -> Result { + if let Some(record) = store + .get_latest_policy(sandbox.object_id()) + .await + .map_err(|e| Status::internal(format!("fetch latest policy failed: {e}")))? + { + return ProtoSandboxPolicy::decode(record.policy_payload.as_slice()) + .map_err(|e| Status::internal(format!("decode current policy failed: {e}"))); + } + Ok(sandbox + .spec + .as_ref() + .and_then(|spec| spec.policy.clone()) + .unwrap_or_default()) +} + +pub(super) async fn validate_candidate_provider_attachments( + state: &ServerState, + workspace: &str, + sandbox: &Sandbox, + provider_names: &[String], +) -> Result<(), Status> { + let base_policy = current_base_policy_for_sandbox(state.store.as_ref(), sandbox).await?; + let provider_layers = + provider_policy_layers_for_sandbox(state, workspace, sandbox, provider_names).await?; + validate_candidate_effective_policy(&base_policy, &provider_layers) +} + +pub(super) async fn provider_policy_composition_enabled(store: &Store) -> Result { + let global_settings = load_global_settings(store).await?; + Ok( + decode_policy_from_global_settings(&global_settings)?.is_none() + && bool_setting_enabled(&global_settings, settings::PROVIDERS_V2_ENABLED_KEY)?, + ) +} + fn truncate_for_log(input: &str, max_chars: usize) -> String { let mut chars = input.chars(); let truncated: String = chars.by_ref().take(max_chars).collect(); @@ -1765,6 +1880,7 @@ async fn handle_update_config_inner( validate_policy_safety(&new_policy)?; crate::middleware::validate_policy(state.middleware_registry.as_ref(), &new_policy) .await?; + validate_candidate_effective_policy(&new_policy, &[])?; let payload = new_policy.encode_to_vec(); let hash = deterministic_policy_hash(&new_policy); @@ -2031,6 +2147,9 @@ async fn handle_update_config_inner( .ok_or_else(|| Status::internal("sandbox has no spec"))?; let merge_ops = parse_merge_operations(&req.merge_operations)?; validate_merge_operations_for_server(&merge_ops)?; + let provider_layers = + provider_policy_layers_for_sandbox(state, &workspace, &sandbox, &spec.providers) + .await?; let atomic_context = AtomicPolicyWriteContext { expected_resource_version: req.expected_resource_version, provenance: &req.annotations, @@ -2046,6 +2165,7 @@ async fn handle_update_config_inner( &workspace, baseline_policy.as_ref(), &merge_ops, + &provider_layers, Some(&atomic_context), ) .await?; @@ -2149,6 +2269,9 @@ async fn handle_update_config_inner( validate_policy_safety(&new_policy)?; crate::middleware::validate_policy(state.middleware_registry.as_ref(), &new_policy).await?; + let provider_layers = + provider_policy_layers_for_sandbox(state, &workspace, &sandbox, &spec.providers).await?; + validate_candidate_effective_policy(&new_policy, &provider_layers)?; let _sandbox_sync_guard = if backfill_policy.is_some() { Some(state.compute.sandbox_sync_guard().await) @@ -3068,8 +3191,21 @@ async fn handle_approve_draft_chunk_inner( "ApproveDraftChunk: merging rule into active policy" ); - let (version, hash) = - merge_chunk_into_policy(state.store.as_ref(), &sandbox_id, &workspace, &chunk).await?; + let provider_names = sandbox + .spec + .as_ref() + .map(|spec| spec.providers.as_slice()) + .unwrap_or_default(); + let provider_layers = + provider_policy_layers_for_sandbox(state, &workspace, &sandbox, provider_names).await?; + let (version, hash) = merge_chunk_into_policy( + state.store.as_ref(), + &sandbox_id, + &workspace, + &chunk, + &provider_layers, + ) + .await?; let chunk_summary = summarize_draft_chunk_rule(&chunk)?; let now_ms = current_time_ms(); @@ -3282,6 +3418,33 @@ async fn handle_approve_all_draft_chunks_inner( let mut chunks_skipped: u32 = 0; let mut last_version: i64 = 0; let mut last_hash = String::new(); + let provider_names = sandbox + .spec + .as_ref() + .map(|spec| spec.providers.as_slice()) + .unwrap_or_default(); + let provider_layers = + provider_policy_layers_for_sandbox(state, &workspace, &sandbox, provider_names).await?; + let mut bulk_candidate = + current_base_policy_for_sandbox(state.store.as_ref(), &sandbox).await?; + for chunk in &pending_chunks { + let security_notes = current_draft_chunk_security_notes(chunk)?; + if !req.include_security_flagged && !security_notes.is_empty() { + continue; + } + let rule = NetworkPolicyRule::decode(chunk.proposed_rule.as_slice()) + .map_err(|e| Status::internal(format!("decode proposed_rule failed: {e}")))?; + let operations = [PolicyMergeOp::AddRule { + rule_name: chunk.rule_name.clone(), + rule, + }]; + validate_merge_operations_for_server(&operations)?; + bulk_candidate = merge_policy(bulk_candidate, &operations) + .map_err(map_policy_merge_error)? + .policy; + } + validate_policy_safety(&bulk_candidate)?; + validate_candidate_effective_policy(&bulk_candidate, &provider_layers)?; for chunk in &pending_chunks { let security_notes = current_draft_chunk_security_notes(chunk)?; @@ -3306,8 +3469,14 @@ async fn handle_approve_all_draft_chunks_inner( "ApproveAllDraftChunks: merging chunk" ); - let (version, hash) = - merge_chunk_into_policy(state.store.as_ref(), &sandbox_id, &workspace, chunk).await?; + let (version, hash) = merge_chunk_into_policy( + state.store.as_ref(), + &sandbox_id, + &workspace, + chunk, + &provider_layers, + ) + .await?; last_version = version; last_hash = hash; let chunk_summary = summarize_draft_chunk_rule(chunk)?; @@ -4179,6 +4348,7 @@ async fn apply_merge_operations_with_retry( workspace: &str, baseline_policy: Option<&ProtoSandboxPolicy>, operations: &[PolicyMergeOp], + provider_layers: &[ProviderPolicyLayer], atomic_context: Option<&AtomicPolicyWriteContext<'_>>, ) -> Result<(i64, String, Option), Status> { for attempt in 1..=MERGE_RETRY_LIMIT { @@ -4202,6 +4372,7 @@ async fn apply_merge_operations_with_retry( validate_static_fields_unchanged(baseline_policy, &new_policy)?; } validate_policy_safety(&new_policy)?; + validate_candidate_effective_policy(&new_policy, provider_layers)?; if let Some(ref current) = latest && current.policy_hash == hash @@ -4297,6 +4468,7 @@ pub(super) async fn merge_chunk_into_policy( sandbox_id: &str, workspace: &str, chunk: &DraftChunkRecord, + provider_layers: &[ProviderPolicyLayer], ) -> Result<(i64, String), Status> { let rule = NetworkPolicyRule::decode(chunk.proposed_rule.as_slice()) .map_err(|e| Status::internal(format!("decode proposed_rule failed: {e}")))?; @@ -4305,9 +4477,17 @@ pub(super) async fn merge_chunk_into_policy( rule, }]; validate_merge_operations_for_server(&operations)?; - apply_merge_operations_with_retry(store, sandbox_id, workspace, None, &operations, None) - .await - .map(|(version, hash, _)| (version, hash)) + apply_merge_operations_with_retry( + store, + sandbox_id, + workspace, + None, + &operations, + provider_layers, + None, + ) + .await + .map(|(version, hash, _)| (version, hash)) } async fn remove_chunk_from_policy( @@ -4325,6 +4505,7 @@ async fn remove_chunk_from_policy( rule_name: chunk.rule_name.clone(), binary_path: chunk.binary.clone(), }], + &[], None, ) .await @@ -5432,6 +5613,14 @@ mod tests { } } + fn test_ambiguous_policy() -> ProtoSandboxPolicy { + let mut left = test_policy_with_rule("left", "api.example.com"); + left.network_policies.get_mut("left").unwrap().endpoints[0].tls = "skip".to_string(); + let right = test_policy_with_rule("right", "api.example.com"); + left.network_policies.extend(right.network_policies); + left + } + fn test_sandbox( id: &str, name: &str, @@ -6134,6 +6323,171 @@ mod tests { ); } + #[test] + fn candidate_effective_policy_rejects_provider_endpoint_ambiguity() { + let base = test_policy_with_rule("base", "api.example.com"); + let mut provider_rule = test_policy_with_rule("provider", "api.example.com") + .network_policies + .remove("provider") + .unwrap(); + provider_rule.endpoints[0].tls = "skip".to_string(); + let layers = [ProviderPolicyLayer { + rule_name: "_provider_test".to_string(), + rule: provider_rule, + }]; + + let error = validate_candidate_effective_policy(&base, &layers) + .expect_err("provider composition must reject endpoint ambiguity"); + assert_eq!(error.code(), Code::FailedPrecondition); + assert!(error.message().contains("api.example.com")); + assert!(error.message().contains("tls")); + } + + #[tokio::test] + async fn update_config_rejects_ambiguous_policy_before_persisting_revision() { + let state = test_server_state().await; + let mut sandbox = test_sandbox( + "sb-ambiguous-update", + "ambiguous-update", + ProtoSandboxPolicy::default(), + Vec::new(), + ); + sandbox.spec.as_mut().unwrap().policy = None; + state.store.put_message(&sandbox).await.unwrap(); + + let error = handle_update_config( + &state, + with_user(Request::new(UpdateConfigRequest { + name: "ambiguous-update".to_string(), + workspace: "default".to_string(), + policy: Some(test_ambiguous_policy()), + ..Default::default() + })), + ) + .await + .expect_err("ambiguous policy must fail before persistence"); + + assert_eq!(error.code(), Code::FailedPrecondition); + assert!(error.message().contains("ambiguity validation failed")); + assert!( + state + .store + .get_latest_policy("sb-ambiguous-update") + .await + .unwrap() + .is_none(), + "invalid policy must not leave a revision in history" + ); + } + + #[tokio::test] + async fn merge_operations_reject_ambiguity_before_persisting_revision() { + let state = test_server_state().await; + let mut policy = test_ambiguous_policy(); + policy.network_policies.get_mut("left").unwrap().endpoints[0].path = "/v1/*".to_string(); + policy.network_policies.get_mut("right").unwrap().endpoints[0].path = + "/v1/users".to_string(); + let operations = policy + .network_policies + .into_iter() + .map(|(rule_name, rule)| PolicyMergeOp::AddRule { rule_name, rule }) + .collect::>(); + + let error = apply_merge_operations_with_retry( + state.store.as_ref(), + "sb-ambiguous-merge", + "default", + None, + &operations, + &[], + None, + ) + .await + .expect_err("ambiguous merge must fail before persistence"); + + assert_eq!(error.code(), Code::FailedPrecondition); + assert!( + state + .store + .get_latest_policy("sb-ambiguous-merge") + .await + .unwrap() + .is_none() + ); + } + + #[tokio::test] + async fn provider_attachment_preflight_rejects_composed_ambiguity() { + use openshell_core::proto::{ + ProviderProfile, ProviderProfileCategory, StoredProviderProfile, + }; + + let state = test_server_state().await; + enable_providers_v2(&state).await; + state + .store + .put_message(&StoredProviderProfile { + metadata: Some(openshell_core::proto::datamodel::v1::ObjectMeta { + id: "profile-ambiguous".to_string(), + name: "ambiguous".to_string(), + created_at_ms: 1_000_000, + labels: HashMap::new(), + resource_version: 0, + annotations: HashMap::new(), + workspace: "default".to_string(), + deletion_timestamp_ms: 0, + }), + profile: Some(ProviderProfile { + id: "ambiguous".to_string(), + display_name: "Ambiguous".to_string(), + category: ProviderProfileCategory::Other as i32, + endpoints: vec![NetworkEndpoint { + host: "api.example.com".to_string(), + port: 443, + tls: "skip".to_string(), + ..Default::default() + }], + ..Default::default() + }), + }) + .await + .unwrap(); + state + .store + .put_message(&test_provider("candidate-provider", "ambiguous")) + .await + .unwrap(); + let sandbox = test_sandbox( + "sb-provider-ambiguity", + "provider-ambiguity", + test_policy_with_rule("base", "api.example.com"), + Vec::new(), + ); + state.store.put_message(&sandbox).await.unwrap(); + + let error = super::super::sandbox::handle_attach_sandbox_provider( + &state, + Request::new(openshell_core::proto::AttachSandboxProviderRequest { + sandbox_name: "provider-ambiguity".to_string(), + provider_name: "candidate-provider".to_string(), + expected_resource_version: 0, + workspace: "default".to_string(), + }), + ) + .await + .expect_err("provider attachment must validate the composed policy"); + + assert_eq!(error.code(), Code::FailedPrecondition); + assert!(error.message().contains("tls")); + let stored = state + .store + .get_message_by_name::("default", "provider-ambiguity") + .await + .unwrap() + .unwrap(); + assert!(stored.spec.unwrap().providers.is_empty()); + } + #[tokio::test] async fn sandbox_config_rejects_invalid_provider_composed_policy() { use openshell_core::proto::{ @@ -10850,9 +11204,10 @@ mod tests { rejection_reason: String::new(), }; - let (version, _) = merge_chunk_into_policy(&store, &chunk.sandbox_id, "default", &chunk) - .await - .unwrap(); + let (version, _) = + merge_chunk_into_policy(&store, &chunk.sandbox_id, "default", &chunk, &[]) + .await + .unwrap(); assert_eq!(version, 1); @@ -10947,7 +11302,7 @@ mod tests { rejection_reason: String::new(), }; - let (version, _) = merge_chunk_into_policy(&store, sandbox_id, "default", &chunk) + let (version, _) = merge_chunk_into_policy(&store, sandbox_id, "default", &chunk, &[]) .await .unwrap(); assert_eq!(version, 2); @@ -11049,7 +11404,7 @@ mod tests { rejection_reason: String::new(), }; - let (version, _) = merge_chunk_into_policy(&store, sandbox_id, "default", &chunk) + let (version, _) = merge_chunk_into_policy(&store, sandbox_id, "default", &chunk, &[]) .await .unwrap(); assert_eq!(version, 2); @@ -11131,9 +11486,23 @@ mod tests { let (left, right) = tokio::join!( apply_merge_operations_with_retry( - &store, sandbox_id, "default", None, &add_allow, None + &store, + sandbox_id, + "default", + None, + &add_allow, + &[], + None + ), + apply_merge_operations_with_retry( + &store, + sandbox_id, + "default", + None, + &add_deny, + &[], + None ), - apply_merge_operations_with_retry(&store, sandbox_id, "default", None, &add_deny, None), ); let mut versions = vec![left.unwrap().0, right.unwrap().0]; @@ -11520,6 +11889,34 @@ mod tests { assert!(err.message().contains("reserved '_provider_' prefix")); } + #[tokio::test] + async fn update_config_global_policy_rejects_ambiguity_before_persisting() { + let state = test_server_state().await; + + let error = handle_update_config( + &state, + with_user(Request::new(UpdateConfigRequest { + global: true, + policy: Some(test_ambiguous_policy()), + ..Default::default() + })), + ) + .await + .expect_err("ambiguous global policy must fail before persistence"); + + assert_eq!(error.code(), Code::FailedPrecondition); + assert!( + state + .store + .get_latest_policy(GLOBAL_POLICY_SANDBOX_ID) + .await + .unwrap() + .is_none() + ); + let settings = load_global_settings(state.store.as_ref()).await.unwrap(); + assert!(!settings.settings.contains_key(POLICY_SETTING_KEY)); + } + #[test] fn merge_effective_settings_global_overrides_sandbox_key() { let global = StoredSettings { diff --git a/crates/openshell-server/src/grpc/provider.rs b/crates/openshell-server/src/grpc/provider.rs index 72f98add75..2334044e12 100644 --- a/crates/openshell-server/src/grpc/provider.rs +++ b/crates/openshell-server/src/grpc/provider.rs @@ -20,6 +20,7 @@ use openshell_core::proto::{ use openshell_core::telemetry::{ LifecycleOperation, ProviderProfile as TelemetryProviderProfile, TelemetryOutcome, }; +use openshell_policy::ProviderPolicyLayer; use prost::Message; use std::collections::HashMap; use tonic::Status; @@ -2193,12 +2194,12 @@ async fn profile_attached_sandbox_diagnostics( profiles: &[(String, ProviderTypeProfile)], operation: &str, ) -> Result, Status> { - let mut candidate_profiles = HashMap::::new(); + let mut candidate_profiles = HashMap::::new(); for (source, profile) in profiles { let Some(id) = normalize_profile_id(&profile.id) else { continue; }; - candidate_profiles.insert(id, (source.clone(), profile.to_proto())); + candidate_profiles.insert(id, (source.clone(), profile.clone())); } if candidate_profiles.is_empty() { return Ok(Vec::new()); @@ -2226,11 +2227,14 @@ async fn profile_attached_sandbox_diagnostics( .await? }; let mut diagnostics = Vec::new(); + let validate_policy_composition = + super::policy::provider_policy_composition_enabled(store).await?; for sandbox in sandboxes { let sandbox_name = sandbox.object_name().to_string(); let sandbox_workspace = sandbox.object_workspace().to_string(); let spec = sandbox.spec.as_ref().expect("filtered by scan_sandboxes"); let mut bindings = Vec::new(); + let mut provider_layers = Vec::new(); let mut imported_profiles_used = Vec::<(String, String)>::new(); for provider_name in &spec.providers { @@ -2246,21 +2250,41 @@ async fn profile_attached_sandbox_diagnostics( else { continue; }; + let profile_id = + normalize_provider_type(&provider.r#type).unwrap_or(provider.r#type.as_str()); let scope_mismatch = (is_platform_scope && !provider.profile_workspace.is_empty()) || (!is_platform_scope && provider.profile_workspace.is_empty()); if scope_mismatch { bindings.extend(dynamic_token_grant_bindings_for_provider_with_catalog( catalog, &provider, )); + if validate_policy_composition + && let Some(profile) = get_provider_type_profile_for_scope( + catalog, + profile_id, + &provider.profile_workspace, + ) + { + let rule_name = openshell_policy::provider_rule_name(provider.object_name()); + provider_layers.push(ProviderPolicyLayer { + rule: profile.network_policy_rule(&rule_name), + rule_name, + }); + } continue; } - let profile_id = - normalize_provider_type(&provider.r#type).unwrap_or(provider.r#type.as_str()); if let Some((source, profile)) = candidate_profiles.get(profile_id) { bindings.extend(dynamic_token_grant_bindings_for_profile( provider.object_name(), - profile, + &profile.to_proto(), )); + if validate_policy_composition { + let rule_name = openshell_policy::provider_rule_name(provider.object_name()); + provider_layers.push(ProviderPolicyLayer { + rule: profile.network_policy_rule(&rule_name), + rule_name, + }); + } let used = (source.clone(), profile_id.to_string()); if !imported_profiles_used.contains(&used) { imported_profiles_used.push(used); @@ -2269,6 +2293,19 @@ async fn profile_attached_sandbox_diagnostics( bindings.extend(dynamic_token_grant_bindings_for_provider_with_catalog( catalog, &provider, )); + if validate_policy_composition + && let Some(profile) = get_provider_type_profile_for_scope( + catalog, + profile_id, + &provider.profile_workspace, + ) + { + let rule_name = openshell_policy::provider_rule_name(provider.object_name()); + provider_layers.push(ProviderPolicyLayer { + rule: profile.network_policy_rule(&rule_name), + rule_name, + }); + } } } @@ -2289,6 +2326,27 @@ async fn profile_attached_sandbox_diagnostics( }); } } + if validate_policy_composition { + let base_policy = + super::policy::current_base_policy_for_sandbox(store, &sandbox).await?; + if let Err(error) = + super::policy::validate_candidate_effective_policy(&base_policy, &provider_layers) + { + for (source, profile_id) in &imported_profiles_used { + diagnostics.push(ProfileValidationDiagnostic { + source: source.clone(), + profile_id: profile_id.clone(), + field: "endpoints".to_string(), + message: format!( + "{operation} would create ambiguous network endpoints on sandbox \ + '{sandbox_name}': {}", + error.message() + ), + severity: "error".to_string(), + }); + } + } + } } Ok(diagnostics) @@ -3063,10 +3121,11 @@ mod tests { GetProviderProfileRequest, GetProviderRefreshStatusRequest, GetProviderRequest, ImportProviderProfilesRequest, L7Allow, L7Rule, LintProviderProfilesRequest, ListProviderProfilesRequest, ListProvidersRequest, NetworkBinary, NetworkEndpoint, + NetworkPolicyRule, ProviderCredentialRefresh, ProviderCredentialRefreshMaterial, ProviderCredentialTokenGrant, ProviderCredentialTokenGrantAudienceOverride, ProviderProfile, ProviderProfileCategory, ProviderProfileCredential, ProviderProfileImportItem, RotateProviderCredentialRequest, - Sandbox, SandboxSpec, StoredProviderProfile, UpdateProviderProfilesRequest, + Sandbox, SandboxPolicy, SandboxSpec, StoredProviderProfile, UpdateProviderProfilesRequest, UpdateProviderRequest, }; use openshell_core::{ObjectId, ObjectName}; @@ -4015,6 +4074,135 @@ mod tests { assert_eq!(fetched.id, "custom-api"); } + #[tokio::test] + async fn profile_update_rejects_fanout_endpoint_ambiguity_without_persisting() { + let state = test_server_state().await; + crate::grpc::policy::save_global_settings( + state.store.as_ref(), + &crate::grpc::StoredSettings { + revision: 1, + settings: std::iter::once(( + openshell_core::settings::PROVIDERS_V2_ENABLED_KEY.to_string(), + crate::grpc::StoredSettingValue::Bool(true), + )) + .collect(), + ..Default::default() + }, + ) + .await + .unwrap(); + + let mut initial_profile = custom_profile("fanout-ambiguity"); + initial_profile.endpoints.push(NetworkEndpoint { + host: "other.example.com".to_string(), + port: 443, + ..Default::default() + }); + let imported = handle_import_provider_profiles( + &state, + Request::new(ImportProviderProfilesRequest { + profiles: vec![ProviderProfileImportItem { + profile: Some(initial_profile), + source: "fanout.yaml".to_string(), + }], + workspace: "default".to_string(), + }), + ) + .await + .unwrap() + .into_inner(); + assert!(imported.imported); + let resource_version = imported.profiles[0].resource_version; + + create_provider_record( + state.store.as_ref(), + "default", + provider_with_values("fanout-provider", "fanout-ambiguity"), + ) + .await + .unwrap(); + state + .store + .put_message(&Sandbox { + metadata: Some(openshell_core::proto::datamodel::v1::ObjectMeta { + id: "fanout-sandbox-id".to_string(), + name: "fanout-sandbox".to_string(), + created_at_ms: 0, + labels: HashMap::new(), + resource_version: 0, + annotations: HashMap::new(), + workspace: "default".to_string(), + deletion_timestamp_ms: 0, + }), + spec: Some(SandboxSpec { + providers: vec!["fanout-provider".to_string()], + policy: Some(SandboxPolicy { + network_policies: HashMap::from([( + "base".to_string(), + NetworkPolicyRule { + name: "base".to_string(), + endpoints: vec![NetworkEndpoint { + host: "api.example.com".to_string(), + port: 443, + ..Default::default() + }], + ..Default::default() + }, + )]), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + }) + .await + .unwrap(); + + let mut conflicting_profile = custom_profile("fanout-ambiguity"); + conflicting_profile.resource_version = resource_version; + conflicting_profile.endpoints.push(NetworkEndpoint { + host: "api.example.com".to_string(), + port: 443, + tls: "skip".to_string(), + ..Default::default() + }); + let response = handle_update_provider_profiles( + &state, + Request::new(UpdateProviderProfilesRequest { + profile: Some(ProviderProfileImportItem { + profile: Some(conflicting_profile), + source: "fanout.yaml".to_string(), + }), + expected_resource_version: resource_version, + id: "fanout-ambiguity".to_string(), + workspace: "default".to_string(), + }), + ) + .await + .unwrap() + .into_inner(); + + assert!(!response.updated); + assert!(response.diagnostics.iter().any(|diagnostic| { + diagnostic.field == "endpoints" + && diagnostic.message.contains("fanout-sandbox") + && diagnostic.message.contains("tls") + })); + let stored = handle_get_provider_profile( + &state, + Request::new(GetProviderProfileRequest { + id: "fanout-ambiguity".to_string(), + workspace: "default".to_string(), + }), + ) + .await + .unwrap() + .into_inner() + .profile + .unwrap(); + assert_eq!(stored.endpoints[0].host, "other.example.com"); + } + #[tokio::test] async fn import_provider_profile_rejects_builtin_overwrite() { let state = test_server_state().await; diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index 405932f037..043774f655 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -504,6 +504,13 @@ pub(super) async fn handle_attach_sandbox_provider( &candidate_spec.providers, ) .await?; + super::policy::validate_candidate_provider_attachments( + state, + &workspace, + &sandbox, + &candidate_spec.providers, + ) + .await?; let provider_name = request.provider_name.clone(); let attached = Arc::new(AtomicBool::new(false)); diff --git a/e2e/rust/tests/proxy_egress_pipeline.rs b/e2e/rust/tests/proxy_egress_pipeline.rs index 35049bc74b..cd33ffc6e9 100644 --- a/e2e/rust/tests/proxy_egress_pipeline.rs +++ b/e2e/rust/tests/proxy_egress_pipeline.rs @@ -974,8 +974,7 @@ async fn policy_reload_updates_both_adapters_and_closes_existing_http_tunnel() { } #[tokio::test] -#[allow(clippy::too_many_lines)] -async fn ambiguous_policy_update_fails_closed_without_contacting_upstream() { +async fn ambiguous_policy_update_is_rejected_without_replacing_active_policy() { let server = KeepAliveHttpServer::start() .await .expect("start keep-alive HTTP server"); @@ -1014,6 +1013,9 @@ async fn ambiguous_policy_update_fails_closed_without_contacting_upstream() { let before = parse_json_line(&before); assert_eq!(before["connect"], 200, "CONNECT before update: {before}"); assert_eq!(before["forward"], 200, "forward before update: {before}"); + let history_before = run_cli(&["policy", "list", &guard.name]) + .await + .expect("list policy history before rejected update"); let connections_before_rejection = server.connection_count(); let update_error = run_cli(&[ @@ -1027,77 +1029,36 @@ async fn ambiguous_policy_update_fails_closed_without_contacting_upstream() { "120", ]) .await - .expect_err("ambiguous policy must report a failed revision"); + .expect_err("ambiguous policy must be rejected before persistence"); assert!( update_error.contains("ambiguity validation failed"), "policy update should explain the ambiguity:\n{update_error}" ); - let quarantined = guard - .exec(&["python3", "-c", &status_script]) + let history_after = run_cli(&["policy", "list", &guard.name]) .await - .expect("exercise both adapters in fail-closed quarantine"); - let quarantined = parse_json_line(&quarantined); - assert_eq!( - quarantined["connect"], 403, - "CONNECT in quarantine: {quarantined}" - ); + .expect("list policy history after rejected update"); assert_eq!( - quarantined["forward"], 403, - "forward in quarantine: {quarantined}" - ); - tokio::time::sleep(std::time::Duration::from_millis(200)).await; - assert_eq!( - server.connection_count(), - connections_before_rejection, - "quarantined requests must not contact the upstream server" - ); - - let logs = wait_for_sandbox_logs(&guard.name, |logs| { - logs.contains("configured_mode=fail_closed effective_mode=fail_closed") - && logs.contains("previous policy IS NOT active") - && logs.contains("conflicting metadata") - && logs.contains("tls") - }) - .await - .expect("fetch sandbox logs after rejection"); - assert!( - logs.contains("configured_mode=fail_closed effective_mode=fail_closed"), - "OCSF logs should state the effective validation posture:\n{logs}" - ); - assert!( - logs.contains("previous policy IS NOT active"), - "OCSF logs should state that the previous policy is inactive:\n{logs}" - ); - assert!( - logs.contains("conflicting metadata") && logs.contains("tls"), - "OCSF logs should contain the overlap rationale:\n{logs}" + history_after, history_before, + "rejected policy must not create a revision" ); - run_cli(&[ - "policy", - "set", - &guard.name, - "--policy", - &valid_policy_path, - "--wait", - "--timeout", - "120", - ]) - .await - .expect("valid policy should exit quarantine"); - let recovered = guard + let after_rejection = guard .exec(&["python3", "-c", &status_script]) .await - .expect("exercise both adapters after recovery"); - let recovered = parse_json_line(&recovered); + .expect("exercise both adapters after rejected update"); + let after_rejection = parse_json_line(&after_rejection); assert_eq!( - recovered["connect"], 200, - "CONNECT after recovery: {recovered}" + after_rejection["connect"], 200, + "CONNECT should keep using the active valid policy: {after_rejection}" ); assert_eq!( - recovered["forward"], 200, - "forward after recovery: {recovered}" + after_rejection["forward"], 200, + "forward HTTP should keep using the active valid policy: {after_rejection}" + ); + assert!( + server.connection_count() > connections_before_rejection, + "active-policy requests should still contact the upstream server" ); guard.cleanup().await; From 9f0996e0d93bd0ebfc7cde093d501f6798347922 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Mon, 27 Jul 2026 11:49:52 -0700 Subject: [PATCH 23/30] docs(policy): explain ambiguity preflight behavior Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- .../skills/debug-openshell-cluster/SKILL.md | 26 +++++++++++++++++++ .../skills/generate-sandbox-policy/SKILL.md | 7 +++-- .../generate-sandbox-policy/examples.md | 4 ++- .agents/skills/openshell-cli/SKILL.md | 14 ++++++++++ architecture/security-policy.md | 15 ++++++++--- docs/reference/gateway-config.mdx | 2 +- docs/reference/policy-schema.mdx | 2 +- docs/sandboxes/policies.mdx | 25 ++++++++++++++---- 8 files changed, 82 insertions(+), 13 deletions(-) diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index 0de85fb0a4..f3975841c9 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -106,6 +106,30 @@ The middleware service must start before the gateway and be reachable from both At request time, distinguish an explicit `middleware_denied` result from `middleware_failed`. A denial is always enforced. A failure follows the policy-local `on_error`: `fail_closed` blocks the request, while `fail_open` bypasses only that stage and emits a detection finding. If a running supervisor cannot install a new registry, it preserves its last-known-good generation and emits a configuration failure event. +For network policy validation failures, first distinguish a gateway mutation +rejection from a supervisor runtime rejection. Direct policy updates, +incremental merges and approvals, provider attachments, and provider-profile +fanout are validated against the complete effective policy before persistence +when the gateway knows the affected sandbox scope. A `FAILED_PRECONDITION` +ambiguity response means no invalid revision or partial fanout was stored. +Supervisor validation remains defense in depth for startup, races, and policy +sources outside those mutation paths. + +Runtime rejection behavior is configured only in `gateway.toml`: + +```toml +[openshell.gateway] +policy_validation_failure_mode = "fail_closed" +``` + +The default `fail_closed` mode deactivates the previous generation, closes +pinned relays, and quarantines new egress until a valid generation loads. +`retain_last_valid` explicitly keeps the previous valid policy active; without +one it still fails closed. Restart the gateway after changing this field. +Inspect sandbox OCSF configuration and finding events for the validation +rationale, configured and effective modes, active generation, and the explicit +`previous_policy_active` state. + ### Step 4: Check Docker-Backed Gateways ```bash @@ -403,6 +427,8 @@ openshell logs | Provider profiles disappear after enabling an interceptor catalog | `provider_profile_sources` selected only an authoritative interceptor or returned invalid/duplicate IDs | Inspect source list and interceptor `Describe`/catalog logs; include `builtin` and `user` when intended | | Gateway fails after registering supervisor middleware | Service unavailable, invalid manifest, duplicate binding, reserved name, or invalid body/timeout limit | Middleware service and gateway logs; `[[openshell.supervisor.middleware]]`; `Describe` response | | Policy update rejects `network_middlewares` | Unknown middleware name, implementation-owned config invalid, duplicate order, broad/invalid host selector, or fail-closed coverage of `tls: skip` | Policy error, gateway logs, middleware `ValidateConfig`, selector and order fields | +| Policy mutation returns `FAILED_PRECONDITION` for endpoint ambiguity | Equally specific effective endpoint selectors disagree on connection or request-processing metadata | CLI error, base and provider-composed policy, affected profile attachments; confirm no new revision was stored | +| Supervisor enters policy quarantine | A runtime candidate failed validation while `policy_validation_failure_mode = "fail_closed"` | Sandbox OCSF config/finding events, validation rationale, active generation, `previous_policy_active` | | HTTP request returns `middleware_failed` or `middleware_denied` | Selected stage failed or explicitly denied the admitted request | Sandbox OCSF logs; policy-local middleware config; service availability; `on_error` | | Custom compute driver is unavailable | Driver process/socket missing, inaccessible, or configured with a reserved/mismatched name | Socket ownership/mode, driver service logs, gateway `GetCapabilities` logs | | Image pull failure | Gateway or sandbox image cannot be pulled | Runtime events and image pull credentials | diff --git a/.agents/skills/generate-sandbox-policy/SKILL.md b/.agents/skills/generate-sandbox-policy/SKILL.md index 3b261b5ded..e2336bd859 100644 --- a/.agents/skills/generate-sandbox-policy/SKILL.md +++ b/.agents/skills/generate-sandbox-policy/SKILL.md @@ -237,7 +237,10 @@ Only needed for the **Moderate** and **Full** tiers. Translate API path paramete | `/api/v1/models/{model_id}/versions/{version}` | `/api/v1/models/*/versions/*` | | All sub-paths under `/api/v1/` | `/api/v1/**` | -Remember: `*` does not cross `/` boundaries. Use `**` for recursive matching across path segments. +Path matching uses the runtime `glob` engine. Both `*` and `**` may cross `/` +boundaries; `?` matches one character, and bracket classes such as `[0-9]` and +`[!0]` are supported. Prefer segment-shaped patterns such as +`/repos/*/issues` for readability, but do not rely on `*` to stop at `/`. ### Building the Explicit Rules List @@ -439,7 +442,7 @@ The policy needs to go somewhere. Determine which mode applies: 2. **Check for conflicts**: - Does a policy with the same key already exist? If so, ask the user whether to **replace** it, **merge** new endpoints/binaries into it, or use a different key. - - Does an existing policy already cover the same host:port? Warn the user — overlapping endpoint coverage across policies causes OPA evaluation errors (complete rule conflict). + - Does an existing endpoint selector overlap the new selector? Compatible overlaps are allowed and can intentionally aggregate allow and deny rules. Reject or revise equally specific overlaps that disagree on connection or request-processing metadata, including TLS, destination constraints, protocol/parser behavior, enforcement, or credential handling. A more-specific path selector may override broader request-processing metadata. 3. **Apply the change**: - **Adding a new policy**: Insert the new policy block under `network_policies`, maintaining the file's existing indentation and style. diff --git a/.agents/skills/generate-sandbox-policy/examples.md b/.agents/skills/generate-sandbox-policy/examples.md index 179e46c00f..b632c176d1 100644 --- a/.agents/skills/generate-sandbox-policy/examples.md +++ b/.agents/skills/generate-sandbox-policy/examples.md @@ -727,7 +727,9 @@ An exact IP is treated as `/32` — only that specific address is permitted. **Agent workflow**: 1. Read `sandbox-policy.yaml` -2. Check that no existing policy already covers `api.github.com:443` — if one does, warn about overlap +2. Check existing selectors for `api.github.com:443`. Compatible overlaps may + aggregate request rules; revise equally specific overlaps that disagree on + TLS, destination, protocol/parser, enforcement, or credential behavior. 3. Check that the key `github_readonly` doesn't already exist 4. Insert the new policy under `network_policies`: diff --git a/.agents/skills/openshell-cli/SKILL.md b/.agents/skills/openshell-cli/SKILL.md index 346672b912..213d55216f 100644 --- a/.agents/skills/openshell-cli/SKILL.md +++ b/.agents/skills/openshell-cli/SKILL.md @@ -354,6 +354,13 @@ Edit `current-policy.yaml` to allow the blocked actions. **For policy content au openshell policy set dev --policy current-policy.yaml --wait ``` +The gateway validates the complete effective candidate—including attached +provider-profile policy—before it stores a direct update, incremental merge, +approved proposal, provider attachment, or profile update that affects attached +sandboxes. An ambiguity failure returns `FAILED_PRECONDITION`; the rejected +candidate does not create a policy revision or partially update affected +sandboxes. Fix the conflicting endpoint selectors and submit again. + The `--wait` flag blocks until the sandbox confirms the policy is loaded (polls every second). Exit codes: - **0**: Policy loaded successfully - **1**: Policy load failed @@ -578,6 +585,13 @@ openshell settings set --global --key providers_v2_enabled --value true Global mutations prompt for confirmation. Use `--yes` only in reviewed automation. +`policy_validation_failure_mode` is gateway startup configuration, not a +mutable `openshell settings` key. Set it under `[openshell.gateway]` in +`gateway.toml` and restart the gateway. The security-first default is +`fail_closed`; `retain_last_valid` is an explicit availability tradeoff. OCSF +configuration events state whether the previous generation is active after a +runtime validation failure. + ## Workflow 10: Service Access Use `forward` for local access and `service` for a gateway-managed HTTP endpoint: diff --git a/architecture/security-policy.md b/architecture/security-policy.md index eeccfc41ee..c68e9a9a1b 100644 --- a/architecture/security-policy.md +++ b/architecture/security-policy.md @@ -108,10 +108,19 @@ connection metadata agrees. When request paths overlap, a path endpoint with a higher specificity rank deterministically overrides broader request-processing metadata. Equally specific overlapping endpoints must agree. +Gateway mutation paths validate the complete effective candidate before +persistence when the affected sandbox scope is known. Direct replacements, +incremental merges and approvals, provider attachment, and profile fanout reject +ambiguity atomically, without creating an invalid revision or partially +activating an update. Supervisor validation remains the defense-in-depth +boundary for startup, concurrent changes, and sources outside those mutations. + The `[openshell.gateway] policy_validation_failure_mode` configuration controls -rejected generations. It defaults to `fail_closed`, which publishes a quarantine -generation, denies new egress, invalidates existing relays, and leaves the -previous policy inactive. Operators may explicitly select +candidates rejected by supervisor runtime validation. Gateway preflight +rejections never become generations and leave the active policy unchanged. The +runtime mode defaults to `fail_closed`, which publishes a quarantine generation, +denies new egress, invalidates existing relays, and leaves the previous policy +inactive. Operators may explicitly select `retain_last_valid`, which keeps the previous generation active. With no previous valid generation, the effective mode remains `fail_closed` regardless of the configured mode. The gateway distributes this startup configuration to diff --git a/docs/reference/gateway-config.mdx b/docs/reference/gateway-config.mdx index 69a1325c58..d09f172322 100644 --- a/docs/reference/gateway-config.mdx +++ b/docs/reference/gateway-config.mdx @@ -180,7 +180,7 @@ phases = ["validate"] Local Docker, Podman, and VM gateways can also set `[openshell.gateway.mtls_auth] enabled = true` to authenticate CLI callers from verified client certificates. Kubernetes deployments must leave this unset and use OIDC or a trusted access proxy; the Helm chart does not render this table. -`[openshell.gateway] policy_validation_failure_mode` controls what sandbox supervisors do when a complete candidate policy fails validation. The default, `fail_closed`, deactivates the previous network policy, closes relays pinned to it, and denies new egress until a valid generation loads. `retain_last_valid` leaves the previous valid generation active. Both modes reject the candidate atomically; startup always fails closed when no previous valid generation exists. Changing the value requires restarting the gateway so it can reload `gateway.toml` and distribute the new posture to sandbox supervisors. +`[openshell.gateway] policy_validation_failure_mode` controls what sandbox supervisors do when a complete candidate policy fails runtime validation. The default, `fail_closed`, deactivates the previous network policy, closes relays pinned to it, and denies new egress until a valid generation loads. `retain_last_valid` leaves the previous valid generation active. Both modes reject the candidate atomically; startup always fails closed when no previous valid generation exists. Gateway mutation paths that can preflight a known effective scope reject invalid candidates before persistence and leave the active policy unchanged regardless of this setting. Changing the value requires restarting the gateway so it can reload `gateway.toml` and distribute the new posture to sandbox supervisors. `[openshell.gateway.gateway_jwt] ttl_secs` controls gateway-minted sandbox JWT lifetime. When omitted, it defaults to `0`: the token `exp` claim and `expires_at_ms` response field become `0`, and the sandbox JWT does not expire. Use that default only for local single-player Docker, Podman, or VM gateways. Kubernetes and other shared deployments should set a positive TTL; Helm renders `3600` seconds by default, and the gateway logs a warning when a Kubernetes gateway uses `0`. diff --git a/docs/reference/policy-schema.mdx b/docs/reference/policy-schema.mdx index 9c4e877eaf..3afc6b9caf 100644 --- a/docs/reference/policy-schema.mdx +++ b/docs/reference/policy-schema.mdx @@ -223,7 +223,7 @@ REST allow rules match HTTP requests by method, path, and optional query paramet | Field | Type | Required | Description | |---|---|---|---| | `method` | string | Yes | HTTP method to allow (for example, `GET`, `POST`). `*` matches any method. | -| `path` | string | Yes | URL path pattern. Supports `*` and `**` glob syntax. | +| `path` | string | Yes | URL path glob. `*` and `**` match zero or more characters and may cross `/`; `?` matches one character; bracket classes such as `[0-9]` and `[!0]` are supported. | | `query` | map | No | Query parameter matchers keyed by decoded param name. Matcher value can be a glob string (`tag: "foo-*"`) or an object with `any` (`tag: { any: ["foo-*", "bar-*"] }`). | Example REST allow rules: diff --git a/docs/sandboxes/policies.mdx b/docs/sandboxes/policies.mdx index 2fae38f909..c116590405 100644 --- a/docs/sandboxes/policies.mdx +++ b/docs/sandboxes/policies.mdx @@ -205,7 +205,21 @@ The following steps outline the hot-reload policy update workflow. OpenShell validates a complete candidate policy before activating any part of it. Endpoints may overlap when their connection and request-processing metadata agree. For example, two `api.example.com:443` REST entries can contribute different allow and deny rules when they use the same TLS, destination, credential, parser, and enforcement settings. A plain L4 endpoint may overlap an L7 endpoint because it authorizes the destination without contributing request-processing metadata. A more-specific path endpoint may override request-processing metadata from a broader endpoint, such as a `/graphql` GraphQL endpoint alongside a general REST endpoint for the same host. OpenShell rejects the candidate when overlapping exact or wildcard host selectors can both contribute equally specific endpoint configuration and disagree on those fields. -The gateway's `policy_validation_failure_mode` configuration determines what happens after rejection. Set it under `[openshell.gateway]` in `gateway.toml`. Its default is `fail_closed`: +When the gateway knows the affected sandbox scope, it validates the complete +effective candidate before persistence. This covers direct policy replacement, +incremental merges and proposal approvals, provider attachment, and +provider-profile updates that fan out to attached sandboxes. An ambiguity +failure returns `FAILED_PRECONDITION`; OpenShell stores no invalid policy +revision and does not partially apply a profile update. Supervisor validation +remains a defense-in-depth boundary for startup, concurrent changes, and policy +sources outside those mutation paths. + +A gateway preflight rejection leaves the currently active policy unchanged +regardless of failure mode because the candidate is never persisted or +distributed. If a candidate reaches a supervisor and fails runtime validation, +the gateway's `policy_validation_failure_mode` configuration determines the +supervisor posture. Set it under `[openshell.gateway]` in `gateway.toml`. Its +default is `fail_closed`: ```toml [openshell.gateway] @@ -354,13 +368,14 @@ means: - match the endpoint `api.github.com:443`. - match HTTP method `POST`. - match paths like `/repos/acme/issues`. -- do not match deeper paths like `/repos/acme/project/issues/123` because `*` matches one path segment. +- also match deeper paths when the surrounding literals align, because `*` may include `/`. Path globs follow the same semantics as YAML allow and deny rules: -- `*` matches one path segment. -- `**` matches any number of segments. -- `/repos/*/issues` matches one repository owner or name segment in the middle. +- `*` and `**` match zero or more characters and may cross `/` boundaries. +- `?` matches exactly one character. +- bracket classes such as `[0-9]` and negated classes such as `[!0]` are supported. +- `/repos/*/issues` matches any intervening text, including multiple path segments. - `/repos/**` matches everything under `/repos/`. The rule-level commands only modify method and path constraints. They do not change binaries, hostnames, ports, protocol settings, or WebSocket message payload matching. From c811382b6160c0f98141ce5210cb5f6cbf17f45c Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Mon, 27 Jul 2026 13:38:59 -0700 Subject: [PATCH 24/30] fix(policy): compare body limits within protocol Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- crates/openshell-policy/src/ambiguity.rs | 39 +++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/crates/openshell-policy/src/ambiguity.rs b/crates/openshell-policy/src/ambiguity.rs index 661948b279..05f8744855 100644 --- a/crates/openshell-policy/src/ambiguity.rs +++ b/crates/openshell-policy/src/ambiguity.rs @@ -264,7 +264,7 @@ fn request_pipeline_conflicts(left: &NetworkEndpoint, right: &NetworkEndpoint) - &normalized_body_limit(right.graphql_max_body_bytes), ); } - if is_json_rpc_family(&left.protocol) && is_json_rpc_family(&right.protocol) { + if left.protocol.eq_ignore_ascii_case(&right.protocol) && is_json_rpc_family(&left.protocol) { push_conflict( &mut conflicts, "json_rpc_max_body_bytes", @@ -896,6 +896,43 @@ mod tests { ); } + #[test] + fn json_rpc_body_limit_is_compared_only_within_the_same_protocol() { + let mut json_rpc = endpoint("api.example.com", 443); + json_rpc.protocol = "json-rpc".to_string(); + json_rpc.json_rpc_max_body_bytes = 1_024; + + let mut mcp = endpoint("api.example.com", 443); + mcp.protocol = "mcp".to_string(); + mcp.json_rpc_max_body_bytes = 2_048; + + let mixed_protocol = find_endpoint_ambiguities(&policy_with(json_rpc, mcp.clone())); + assert_eq!(mixed_protocol.len(), 1); + assert!( + mixed_protocol[0] + .conflicts + .iter() + .any(|field| field.contains("protocol")) + ); + assert!( + !mixed_protocol[0] + .conflicts + .iter() + .any(|field| field.contains("json_rpc_max_body_bytes")) + ); + + let mut other_mcp = mcp.clone(); + other_mcp.json_rpc_max_body_bytes = 4_096; + let same_protocol = find_endpoint_ambiguities(&policy_with(mcp, other_mcp)); + assert_eq!(same_protocol.len(), 1); + assert!( + same_protocol[0] + .conflicts + .iter() + .any(|field| field.contains("json_rpc_max_body_bytes")) + ); + } + #[test] fn websocket_graphql_classification_conflict_is_rejected() { let mut graphql = endpoint("api.example.com", 443); From 966a7761b83086a005e26863efe82a564d295f80 Mon Sep 17 00:00:00 2001 From: John Myers Date: Mon, 27 Jul 2026 19:43:49 -0700 Subject: [PATCH 25/30] test(proxy): avoid global tracing capture race Signed-off-by: John Myers --- .../openshell-supervisor-network/src/proxy.rs | 127 +++++++++++------- .../src/proxy/tests/compatibility.rs | 121 +++++------------ 2 files changed, 115 insertions(+), 133 deletions(-) diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index bc093e439b..917aa1fc7b 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -901,9 +901,18 @@ fn build_forward_policy_deny_ocsf_event( .build() } +fn destination_denial_detail(kind: DestinationDenialKind) -> &'static str { + match kind { + DestinationDenialKind::TrustedGateway => "trusted-gateway check failed", + DestinationDenialKind::InvalidAllowedIps => "invalid allowed_ips in policy", + DestinationDenialKind::AllowedIps => "allowed_ips check failed", + DestinationDenialKind::DeclaredEndpoint => "declared endpoint check failed", + DestinationDenialKind::InternalAddress => "internal address", + } +} + #[allow(clippy::too_many_arguments)] -async fn deny_connect_destination( - client: &mut TcpStream, +fn build_connect_destination_deny_ocsf_event( denial: &DestinationDenial, peer_addr: SocketAddr, host: &str, @@ -912,24 +921,15 @@ async fn deny_connect_destination( pid: &str, ancestors: &str, cmdline: &str, - decision: &EgressDecision, - denial_tx: &Option>, - activity_tx: &Option, -) -> Result<()> { - let detail = match denial.kind { - DestinationDenialKind::TrustedGateway => "trusted-gateway check failed", - DestinationDenialKind::InvalidAllowedIps => "invalid allowed_ips in policy", - DestinationDenialKind::AllowedIps => "allowed_ips check failed", - DestinationDenialKind::DeclaredEndpoint => "declared endpoint check failed", - DestinationDenialKind::InternalAddress => "internal address", - }; +) -> openshell_ocsf::OcsfEvent { + let detail = destination_denial_detail(denial.kind); let message = if denial.kind == DestinationDenialKind::InternalAddress { format!("CONNECT blocked: internal address {host}:{port}") } else { format!("CONNECT blocked: {detail} for {host}:{port}") }; - let event = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Open) .action(ActionId::Denied) .disposition(DispositionId::Blocked) @@ -941,8 +941,68 @@ async fn deny_connect_destination( .firewall_rule("-", "ssrf") .message(message) .status_detail(&denial.reason) - .build(); - ocsf_emit!(event); + .build() +} + +#[allow(clippy::too_many_arguments)] +fn build_forward_destination_deny_ocsf_event( + denial: &DestinationDenial, + peer_addr: SocketAddr, + method: &str, + host: &str, + port: u16, + path: &str, + binary: &str, + pid: &str, + ancestors: &str, + cmdline: &str, + policy: &str, +) -> openshell_ocsf::OcsfEvent { + let detail = destination_denial_detail(denial.kind); + let log_detail = if denial.kind == DestinationDenialKind::InternalAddress { + "internal IP without allowed_ips" + } else { + detail + }; + + HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Other) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .http_request(HttpRequest::new( + method, + OcsfUrl::new("http", host, path, port), + )) + .dst_endpoint(Endpoint::from_domain(host, port)) + .src_endpoint(Endpoint::from_ip(peer_addr.ip(), peer_addr.port())) + .actor_process(Process::from_bypass(binary, pid, ancestors).with_cmd_line(cmdline)) + .firewall_rule(policy, "ssrf") + .message(format!("FORWARD blocked: {log_detail} for {host}:{port}")) + .status_detail(&denial.reason) + .build() +} + +#[allow(clippy::too_many_arguments)] +async fn deny_connect_destination( + client: &mut TcpStream, + denial: &DestinationDenial, + peer_addr: SocketAddr, + host: &str, + port: u16, + binary: &str, + pid: &str, + ancestors: &str, + cmdline: &str, + decision: &EgressDecision, + denial_tx: &Option>, + activity_tx: &Option, +) -> Result<()> { + let detail = destination_denial_detail(denial.kind); + ocsf_emit!(build_connect_destination_deny_ocsf_event( + denial, peer_addr, host, port, binary, pid, ancestors, cmdline, + )); emit_denial( denial_tx, @@ -988,37 +1048,10 @@ async fn deny_forward_destination( denial_tx: Option<&mpsc::UnboundedSender>, activity_tx: Option<&ActivitySender>, ) -> Result<()> { - let detail = match denial.kind { - DestinationDenialKind::TrustedGateway => "trusted-gateway check failed", - DestinationDenialKind::InvalidAllowedIps => "invalid allowed_ips in policy", - DestinationDenialKind::AllowedIps => "allowed_ips check failed", - DestinationDenialKind::DeclaredEndpoint => "declared endpoint check failed", - DestinationDenialKind::InternalAddress => "internal address", - }; - let log_detail = if denial.kind == DestinationDenialKind::InternalAddress { - "internal IP without allowed_ips" - } else { - detail - }; - - let event = HttpActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Other) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .http_request(HttpRequest::new( - method, - OcsfUrl::new("http", host, path, port), - )) - .dst_endpoint(Endpoint::from_domain(host, port)) - .src_endpoint(Endpoint::from_ip(peer_addr.ip(), peer_addr.port())) - .actor_process(Process::from_bypass(binary, pid, ancestors).with_cmd_line(cmdline)) - .firewall_rule(policy, "ssrf") - .message(format!("FORWARD blocked: {log_detail} for {host}:{port}")) - .status_detail(&denial.reason) - .build(); - ocsf_emit!(event); + let detail = destination_denial_detail(denial.kind); + ocsf_emit!(build_forward_destination_deny_ocsf_event( + denial, peer_addr, method, host, port, path, binary, pid, ancestors, cmdline, policy, + )); emit_denial_simple( denial_tx, diff --git a/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs b/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs index 1ffe08d884..331454c468 100644 --- a/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs +++ b/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs @@ -4,24 +4,8 @@ //! Compatibility and regression contracts for the shared proxy egress pipeline. use super::*; -use std::io::Write; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tracing_subscriber::prelude::*; - -#[derive(Clone)] -struct SharedBuffer(Arc>>); - -impl Write for SharedBuffer { - fn write(&mut self, bytes: &[u8]) -> std::io::Result { - self.0.lock().unwrap().extend_from_slice(bytes); - Ok(bytes.len()) - } - - fn flush(&mut self) -> std::io::Result<()> { - Ok(()) - } -} fn allowed_decision(intent: EgressIntent) -> EgressDecision { EgressDecision { @@ -166,75 +150,27 @@ async fn destination_denials_preserve_adapter_specific_wire_contracts() { #[test] fn representative_adapter_denials_preserve_ocsf_fields() { - let captured = Arc::new(Mutex::new(Vec::new())); - let layer = openshell_ocsf::tracing_layers::OcsfJsonlLayer::new(SharedBuffer(captured.clone())); - let subscriber = tracing_subscriber::registry().with(layer); let denial_reason = "target.example resolves to internal address 10.0.0.5"; - tracing::subscriber::with_default(subscriber, || { - tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap() - .block_on(async { - let denial = DestinationDenial { - kind: DestinationDenialKind::InternalAddress, - reason: denial_reason.to_string(), - }; - let peer: SocketAddr = "127.0.0.1:41000".parse().unwrap(); - - let (_app, mut proxy) = tcp_pair().await; - deny_connect_destination( - &mut proxy, - &denial, - peer, - "target.example", - 8443, - "/usr/bin/curl", - "42", - "/usr/bin/sh", - "curl --proxy", - &allowed_decision(EgressIntent::connect("target.example".to_string(), 8443)), - &None, - &None, - ) - .await - .unwrap(); - - let (_app, mut proxy) = tcp_pair().await; - deny_forward_destination( - &mut proxy, - &denial, - peer, - "POST", - "target.example", - 8080, - "/v1/items", - "/usr/bin/curl", - "42", - "/usr/bin/sh", - "curl --proxy", - "proxy_compatibility", - &allowed_decision(EgressIntent::forward_http( - "target.example".to_string(), - 8080, - )), - None, - None, - ) - .await - .unwrap(); - }); - }); - - let bytes = captured.lock().unwrap().clone(); - let events: Vec = String::from_utf8(bytes) - .unwrap() - .lines() - .map(|line| serde_json::from_str(line).unwrap()) - .collect(); - assert_eq!(events.len(), 2); - - let connect = &events[0]; + let denial = DestinationDenial { + kind: DestinationDenialKind::InternalAddress, + reason: denial_reason.to_string(), + }; + let peer: SocketAddr = "127.0.0.1:41000".parse().unwrap(); + + // Build the production events directly rather than routing through the + // global tracing pipeline. Its callsite-interest cache is process-global, + // so parallel tests can otherwise make captured-event assertions flaky. + let connect = serde_json::to_value(build_connect_destination_deny_ocsf_event( + &denial, + peer, + "target.example", + 8443, + "/usr/bin/curl", + "42", + "/usr/bin/sh", + "curl --proxy", + )) + .unwrap(); assert_eq!(connect["class_name"], "Network Activity"); assert_eq!(connect["activity_name"], "Open"); assert_eq!(connect["action"], "Denied"); @@ -253,7 +189,20 @@ fn representative_adapter_denials_preserve_ocsf_fields() { ); assert_eq!(connect["status_detail"], denial_reason); - let forward = &events[1]; + let forward = serde_json::to_value(build_forward_destination_deny_ocsf_event( + &denial, + peer, + "POST", + "target.example", + 8080, + "/v1/items", + "/usr/bin/curl", + "42", + "/usr/bin/sh", + "curl --proxy", + "proxy_compatibility", + )) + .unwrap(); assert_eq!(forward["class_name"], "HTTP Activity"); assert_eq!(forward["activity_name"], "Other"); assert_eq!(forward["action"], "Denied"); From 71565783f2840d438406373230ef9b147dcbd33b Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:00:26 -0700 Subject: [PATCH 26/30] chore(server): format rebased provider tests Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- crates/openshell-server/src/grpc/provider.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/crates/openshell-server/src/grpc/provider.rs b/crates/openshell-server/src/grpc/provider.rs index 2334044e12..6faa7831b9 100644 --- a/crates/openshell-server/src/grpc/provider.rs +++ b/crates/openshell-server/src/grpc/provider.rs @@ -3121,12 +3121,11 @@ mod tests { GetProviderProfileRequest, GetProviderRefreshStatusRequest, GetProviderRequest, ImportProviderProfilesRequest, L7Allow, L7Rule, LintProviderProfilesRequest, ListProviderProfilesRequest, ListProvidersRequest, NetworkBinary, NetworkEndpoint, - NetworkPolicyRule, - ProviderCredentialRefresh, ProviderCredentialRefreshMaterial, ProviderCredentialTokenGrant, - ProviderCredentialTokenGrantAudienceOverride, ProviderProfile, ProviderProfileCategory, - ProviderProfileCredential, ProviderProfileImportItem, RotateProviderCredentialRequest, - Sandbox, SandboxPolicy, SandboxSpec, StoredProviderProfile, UpdateProviderProfilesRequest, - UpdateProviderRequest, + NetworkPolicyRule, ProviderCredentialRefresh, ProviderCredentialRefreshMaterial, + ProviderCredentialTokenGrant, ProviderCredentialTokenGrantAudienceOverride, + ProviderProfile, ProviderProfileCategory, ProviderProfileCredential, + ProviderProfileImportItem, RotateProviderCredentialRequest, Sandbox, SandboxPolicy, + SandboxSpec, StoredProviderProfile, UpdateProviderProfilesRequest, UpdateProviderRequest, }; use openshell_core::{ObjectId, ObjectName}; use tonic::{Code, Request}; From 04c16b08142f9de132b86bc01f59baf7c41ea1c3 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:50:09 -0700 Subject: [PATCH 27/30] test(server): authenticate rebased policy requests Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- crates/openshell-server/src/grpc/policy.rs | 2 +- crates/openshell-server/src/grpc/provider.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index b345a8210b..799840fa79 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -6467,7 +6467,7 @@ mod tests { let error = super::super::sandbox::handle_attach_sandbox_provider( &state, - Request::new(openshell_core::proto::AttachSandboxProviderRequest { + authed_request(openshell_core::proto::AttachSandboxProviderRequest { sandbox_name: "provider-ambiguity".to_string(), provider_name: "candidate-provider".to_string(), expected_resource_version: 0, diff --git a/crates/openshell-server/src/grpc/provider.rs b/crates/openshell-server/src/grpc/provider.rs index 6faa7831b9..b5cf0c258e 100644 --- a/crates/openshell-server/src/grpc/provider.rs +++ b/crates/openshell-server/src/grpc/provider.rs @@ -4099,7 +4099,7 @@ mod tests { }); let imported = handle_import_provider_profiles( &state, - Request::new(ImportProviderProfilesRequest { + authed_request(ImportProviderProfilesRequest { profiles: vec![ProviderProfileImportItem { profile: Some(initial_profile), source: "fanout.yaml".to_string(), @@ -4167,7 +4167,7 @@ mod tests { }); let response = handle_update_provider_profiles( &state, - Request::new(UpdateProviderProfilesRequest { + authed_request(UpdateProviderProfilesRequest { profile: Some(ProviderProfileImportItem { profile: Some(conflicting_profile), source: "fanout.yaml".to_string(), @@ -4189,7 +4189,7 @@ mod tests { })); let stored = handle_get_provider_profile( &state, - Request::new(GetProviderProfileRequest { + authed_request(GetProviderProfileRequest { id: "fanout-ambiguity".to_string(), workspace: "default".to_string(), }), From aa5ee7a82832bb4c6aaffe49f440ed41997cc7ec Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Fri, 31 Jul 2026 10:37:41 -0700 Subject: [PATCH 28/30] fix(sandbox): retain runtime on middleware outage Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- crates/openshell-sandbox/src/lib.rs | 220 +++++++++++++++++++++------- 1 file changed, 163 insertions(+), 57 deletions(-) diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index cdbf7d6894..47fe466084 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -2226,6 +2226,40 @@ enum MiddlewareRegistryStatus { NeedsReconciliation, } +#[derive(Debug)] +enum GatewayRuntimeReloadError { + PolicyValidation(miette::Report), + MiddlewareRegistry(miette::Report), +} + +async fn reload_gateway_policy_runtime( + engine: &OpaEngine, + policy: Option<&openshell_core::proto::SandboxPolicy>, + entrypoint_pid: u32, + desired_services: &[openshell_core::proto::SupervisorMiddlewareService], + middleware_registry_changed: bool, +) -> std::result::Result<(), GatewayRuntimeReloadError> { + match policy { + Some(policy) if middleware_registry_changed => { + let registry = connect_middleware_registry(desired_services) + .await + .map_err(GatewayRuntimeReloadError::MiddlewareRegistry)?; + engine + .reload_policy_and_middleware_from_proto_with_pid(policy, entrypoint_pid, registry) + .map_err(GatewayRuntimeReloadError::PolicyValidation) + } + // Policy-only change: the installed registry already matches the + // delivered service set, so swap the engine alone. This must not + // require middleware reachability. + Some(policy) => engine + .reload_from_proto_with_pid(policy, entrypoint_pid) + .map_err(GatewayRuntimeReloadError::PolicyValidation), + None => Err(GatewayRuntimeReloadError::PolicyValidation( + miette::miette!("runtime reload requires a policy payload but none was returned"), + )), + } +} + /// True when the installed middleware registry no longer matches the desired /// service set and must be rebuilt (reconnecting every delivered service). /// @@ -2652,6 +2686,43 @@ struct RejectedPolicyGeneration { configured_mode: PolicyValidationFailureMode, } +enum GatewayRuntimeFailureDisposition { + PolicyRejected { + error: String, + disposition: PolicyValidationFailureDisposition, + }, + MiddlewareUnavailable { + error: String, + }, +} + +fn apply_gateway_runtime_reload_failure( + engine: &OpaEngine, + failure: GatewayRuntimeReloadError, + configured_mode: PolicyValidationFailureMode, + has_last_valid_policy: bool, + version: u32, +) -> Result { + match failure { + GatewayRuntimeReloadError::PolicyValidation(error) => { + let error = error.to_string(); + let disposition = apply_policy_validation_failure( + engine, + configured_mode, + has_last_valid_policy, + version, + &error, + )?; + Ok(GatewayRuntimeFailureDisposition::PolicyRejected { error, disposition }) + } + GatewayRuntimeReloadError::MiddlewareRegistry(error) => { + Ok(GatewayRuntimeFailureDisposition::MiddlewareUnavailable { + error: error.to_string(), + }) + } + } +} + fn apply_policy_validation_failure( engine: &OpaEngine, configured_mode: PolicyValidationFailureMode, @@ -3026,26 +3097,14 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { if policy_runtime_changed { let pid = ctx.entrypoint_pid.load(Ordering::Acquire); - let runtime_result = match result.policy.as_ref() { - Some(policy) if middleware_registry_changed => { - match connect_middleware_registry(&result.supervisor_middleware_services).await - { - Ok(registry) => ctx - .opa_engine - .reload_policy_and_middleware_from_proto_with_pid( - policy, pid, registry, - ), - Err(error) => Err(error), - } - } - // Policy-only change: the installed registry already matches - // the delivered service set, so swap the engine alone. This - // must not require middleware reachability. - Some(policy) => ctx.opa_engine.reload_from_proto_with_pid(policy, pid), - None => Err(miette::miette!( - "runtime reload requires a policy payload but none was returned" - )), - }; + let runtime_result = reload_gateway_policy_runtime( + &ctx.opa_engine, + result.policy.as_ref(), + pid, + &result.supervisor_middleware_services, + middleware_registry_changed, + ) + .await; match runtime_result { Ok(()) => { @@ -3142,50 +3201,57 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { middleware_registry_status = MiddlewareRegistryStatus::Synchronized; last_failed_runtime_revision = None; } - Err(e) => { + Err(failure) => { let failed_revision = (result.config_revision, result.policy_hash.clone()); if last_failed_runtime_revision.as_ref() != Some(&failed_revision) { - let validation_error = e.to_string(); - ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Other, "failed") - .unmapped("version", serde_json::json!(result.version)) - .unmapped("error", serde_json::json!(e.to_string())) - .message(format!( - "Policy and middleware runtime reload failed, keeping last-known-good runtime [version:{} error:{e}]", - result.version - )) - .build()); - let failure_mode = result.policy_validation_failure_mode; - let disposition = apply_policy_validation_failure( + match apply_gateway_runtime_reload_failure( &ctx.opa_engine, + failure, failure_mode, has_last_valid_policy, result.version, - &validation_error, - )?; - emit_policy_validation_failure( - &disposition, - result.version, - &result.policy_hash, - &validation_error, - ); - rejected_policy_generation = Some(RejectedPolicyGeneration { - version: result.version, - policy_hash: result.policy_hash.clone(), - validation_error: validation_error.clone(), - configured_mode: failure_mode, - }); - if policy_changed - && result.version > 0 - && result.policy_source == PolicySource::Sandbox - { - enqueue_policy_status( - &status_sender, - PolicyStatusUpdate::failed(result.version, validation_error), - ); + )? { + GatewayRuntimeFailureDisposition::PolicyRejected { + error, + disposition, + } => { + emit_policy_validation_failure( + &disposition, + result.version, + &result.policy_hash, + &error, + ); + rejected_policy_generation = Some(RejectedPolicyGeneration { + version: result.version, + policy_hash: result.policy_hash.clone(), + validation_error: error.clone(), + configured_mode: failure_mode, + }); + if policy_changed + && result.version > 0 + && result.policy_source == PolicySource::Sandbox + { + enqueue_policy_status( + &status_sender, + PolicyStatusUpdate::failed(result.version, error), + ); + } + } + GatewayRuntimeFailureDisposition::MiddlewareUnavailable { error } => { + ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .state(StateId::Other, "failed") + .unmapped("version", serde_json::json!(result.version)) + .unmapped("error", serde_json::json!(&error)) + .unmapped("previous_policy_active", serde_json::json!(true)) + .message(format!( + "Supervisor middleware registry unavailable, keeping last-known-good policy runtime active [version:{} error:{error}]", + result.version + )) + .build()); + } } } last_failed_runtime_revision = Some(failed_revision); @@ -3750,6 +3816,46 @@ filesystem_policy: assert_eq!(engine.current_generation(), builtins_generation); } + #[tokio::test] + async fn unavailable_middleware_reload_keeps_last_known_good_runtime_active() { + let engine = OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine"); + install_builtin_middleware_registry(&engine) + .await + .expect("install built-in middleware registry"); + let active_generation = engine.current_generation(); + let unavailable_service = openshell_core::proto::SupervisorMiddlewareService { + name: "unavailable-guard".into(), + grpc_endpoint: "http://127.0.0.1:1".into(), + max_body_bytes: 1024, + ..Default::default() + }; + + let failure = reload_gateway_policy_runtime( + &engine, + Some(&proto_policy_fixture()), + 0, + &[unavailable_service], + true, + ) + .await + .expect_err("unavailable middleware must fail candidate preparation"); + let disposition = apply_gateway_runtime_reload_failure( + &engine, + failure, + PolicyValidationFailureMode::FailClosed, + true, + 2, + ) + .expect("middleware failure handling must succeed"); + + assert!(matches!( + disposition, + GatewayRuntimeFailureDisposition::MiddlewareUnavailable { .. } + )); + assert_eq!(engine.current_generation(), active_generation); + assert!(engine.fail_closed_reason().is_none()); + } + #[test] fn failed_gateway_runtime_snapshot_is_retried_without_revision_change() { let services = Vec::new(); From 286d02719e3ecb7e0126aa5934c535c72b0567b3 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Fri, 31 Jul 2026 10:38:03 -0700 Subject: [PATCH 29/30] fix(server): preflight provider composition activation Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- crates/openshell-server/src/grpc/policy.rs | 234 +++++++++++++++++++-- 1 file changed, 216 insertions(+), 18 deletions(-) diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index 799840fa79..5c5faae8a6 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -1200,10 +1200,75 @@ pub(super) async fn validate_candidate_provider_attachments( pub(super) async fn provider_policy_composition_enabled(store: &Store) -> Result { let global_settings = load_global_settings(store).await?; - Ok( - decode_policy_from_global_settings(&global_settings)?.is_none() - && bool_setting_enabled(&global_settings, settings::PROVIDERS_V2_ENABLED_KEY)?, - ) + provider_policy_composition_enabled_in(&global_settings) +} + +fn provider_policy_composition_enabled_in(settings: &StoredSettings) -> Result { + Ok(decode_policy_from_global_settings(settings)?.is_none() + && bool_setting_enabled(settings, settings::PROVIDERS_V2_ENABLED_KEY)?) +} + +async fn validate_provider_composition_for_existing_sandboxes( + state: &ServerState, +) -> Result<(), Status> { + let mut offset = 0; + let mut catalogs = HashMap::::new(); + + loop { + let sandboxes = state + .store + .list_all_messages::(MAX_PAGE_SIZE, offset) + .await + .map_err(|e| Status::internal(format!("list sandboxes failed: {e}")))?; + let page_len = sandboxes.len(); + + for sandbox in sandboxes { + let provider_names = sandbox + .spec + .as_ref() + .map(|spec| spec.providers.as_slice()) + .unwrap_or_default(); + if provider_names.is_empty() { + continue; + } + + let workspace = sandbox.object_workspace().to_string(); + if !catalogs.contains_key(&workspace) { + let catalog = state + .provider_profile_sources + .snapshot_catalog(state.store.as_ref(), &workspace) + .await?; + catalogs.insert(workspace.clone(), catalog); + } + let catalog = catalogs + .get(&workspace) + .expect("catalog was inserted for sandbox workspace"); + let base_policy = + current_base_policy_for_sandbox(state.store.as_ref(), &sandbox).await?; + let provider_layers = profile_provider_policy_layers_with_catalog( + state.store.as_ref(), + catalog, + &workspace, + provider_names, + ) + .await?; + validate_candidate_effective_policy(&base_policy, &provider_layers).map_err(|error| { + Status::failed_precondition(format!( + "cannot activate provider policy composition: sandbox '{}/{}' has an invalid effective policy: {}", + workspace, + sandbox.object_name(), + error.message() + )) + })?; + } + + if page_len < MAX_PAGE_SIZE as usize { + break; + } + offset = offset.saturating_add(MAX_PAGE_SIZE); + } + + Ok(()) } fn truncate_for_log(input: &str, max_chars: usize) -> String { @@ -1983,21 +2048,10 @@ async fn handle_update_config_inner( } let mut global_settings = load_global_settings(state.store.as_ref()).await?; + let provider_composition_was_enabled = + provider_policy_composition_enabled_in(&global_settings)?; let changed = if req.delete_setting { - let removed = global_settings.settings.remove(key).is_some(); - if removed - && key == POLICY_SETTING_KEY - && let Ok(Some(latest)) = state - .store - .get_latest_policy(GLOBAL_POLICY_SANDBOX_ID) - .await - { - let _ = state - .store - .supersede_older_policies(GLOBAL_POLICY_SANDBOX_ID, latest.version + 1) - .await; - } - removed + global_settings.settings.remove(key).is_some() } else { let setting = req .setting_value @@ -2008,8 +2062,27 @@ async fn handle_update_config_inner( }; if changed { + let provider_composition_is_enabled = + provider_policy_composition_enabled_in(&global_settings)?; + if !provider_composition_was_enabled && provider_composition_is_enabled { + validate_provider_composition_for_existing_sandboxes(state).await?; + } + global_settings.revision = global_settings.revision.wrapping_add(1); save_global_settings(state.store.as_ref(), &global_settings).await?; + + if req.delete_setting + && key == POLICY_SETTING_KEY + && let Ok(Some(latest)) = state + .store + .get_latest_policy(GLOBAL_POLICY_SANDBOX_ID) + .await + { + let _ = state + .store + .supersede_older_policies(GLOBAL_POLICY_SANDBOX_ID, latest.version + 1) + .await; + } } return Ok(update_config_response( @@ -11917,6 +11990,131 @@ mod tests { assert!(!settings.settings.contains_key(POLICY_SETTING_KEY)); } + async fn install_ambiguous_provider_binding(state: &Arc, suffix: &str) { + use openshell_core::proto::{ + ProviderProfile, ProviderProfileCategory, StoredProviderProfile, + }; + + let profile_name = format!("ambiguous-{suffix}"); + let provider_name = format!("provider-{suffix}"); + state + .store + .put_message(&StoredProviderProfile { + metadata: Some(openshell_core::proto::datamodel::v1::ObjectMeta { + id: format!("profile-{suffix}"), + name: profile_name.clone(), + created_at_ms: 1_000_000, + labels: HashMap::new(), + resource_version: 0, + annotations: HashMap::new(), + workspace: "default".to_string(), + deletion_timestamp_ms: 0, + }), + profile: Some(ProviderProfile { + id: profile_name.clone(), + display_name: "Ambiguous".to_string(), + category: ProviderProfileCategory::Other as i32, + endpoints: vec![NetworkEndpoint { + host: "api.example.com".to_string(), + port: 443, + tls: "skip".to_string(), + ..Default::default() + }], + ..Default::default() + }), + }) + .await + .unwrap(); + state + .store + .put_message(&test_provider(&provider_name, &profile_name)) + .await + .unwrap(); + state + .store + .put_message(&test_sandbox( + &format!("sandbox-{suffix}"), + &format!("sandbox-{suffix}"), + test_policy_with_rule("base", "api.example.com"), + vec![provider_name], + )) + .await + .unwrap(); + } + + #[tokio::test] + async fn enabling_provider_composition_rejects_existing_ambiguous_binding() { + let state = test_server_state().await; + install_ambiguous_provider_binding(&state, "enable").await; + + let error = handle_update_config( + &state, + with_user(Request::new(UpdateConfigRequest { + global: true, + setting_key: settings::PROVIDERS_V2_ENABLED_KEY.to_string(), + setting_value: Some(SettingValue { + value: Some(setting_value::Value::BoolValue(true)), + }), + ..Default::default() + })), + ) + .await + .expect_err("provider composition must be validated before activation"); + + assert_eq!(error.code(), Code::FailedPrecondition); + assert!(error.message().contains("sandbox-enable")); + assert!(error.message().contains("tls")); + let settings = load_global_settings(state.store.as_ref()).await.unwrap(); + assert!(!bool_setting_enabled(&settings, settings::PROVIDERS_V2_ENABLED_KEY).unwrap()); + } + + #[tokio::test] + async fn deleting_global_policy_rejects_reactivated_ambiguous_provider_binding() { + let state = test_server_state().await; + install_ambiguous_provider_binding(&state, "delete-policy").await; + + handle_update_config( + &state, + with_user(Request::new(UpdateConfigRequest { + global: true, + policy: Some(test_policy_with_rule("global", "global.example.com")), + ..Default::default() + })), + ) + .await + .expect("global policy should suppress provider composition"); + handle_update_config( + &state, + with_user(Request::new(UpdateConfigRequest { + global: true, + setting_key: settings::PROVIDERS_V2_ENABLED_KEY.to_string(), + setting_value: Some(SettingValue { + value: Some(setting_value::Value::BoolValue(true)), + }), + ..Default::default() + })), + ) + .await + .expect("providers may be enabled while a global policy is active"); + + let error = handle_update_config( + &state, + with_user(Request::new(UpdateConfigRequest { + global: true, + setting_key: POLICY_SETTING_KEY.to_string(), + delete_setting: true, + ..Default::default() + })), + ) + .await + .expect_err("global policy deletion must validate reactivated provider composition"); + + assert_eq!(error.code(), Code::FailedPrecondition); + assert!(error.message().contains("sandbox-delete-policy")); + let settings = load_global_settings(state.store.as_ref()).await.unwrap(); + assert!(settings.settings.contains_key(POLICY_SETTING_KEY)); + } + #[test] fn merge_effective_settings_global_overrides_sandbox_key() { let global = StoredSettings { From 9f4447f4a9cb2e914bd33619bb1820ce431d4b4f Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Fri, 31 Jul 2026 12:16:24 -0700 Subject: [PATCH 30/30] fix(sandbox): distinguish runtime failure transitions Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- crates/openshell-sandbox/src/lib.rs | 90 ++++++++++++++++++++++++++++- 1 file changed, 88 insertions(+), 2 deletions(-) diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 47fe466084..f9555d9f24 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -2232,6 +2232,38 @@ enum GatewayRuntimeReloadError { MiddlewareRegistry(miette::Report), } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum GatewayRuntimeFailureClass { + PolicyValidation, + MiddlewareRegistry, +} + +impl GatewayRuntimeReloadError { + fn class(&self) -> GatewayRuntimeFailureClass { + match self { + Self::PolicyValidation(_) => GatewayRuntimeFailureClass::PolicyValidation, + Self::MiddlewareRegistry(_) => GatewayRuntimeFailureClass::MiddlewareRegistry, + } + } +} + +#[derive(Debug, PartialEq, Eq)] +struct FailedRuntimeRevision { + config_revision: u64, + policy_hash: String, + failure_class: GatewayRuntimeFailureClass, +} + +impl FailedRuntimeRevision { + fn new(config_revision: u64, policy_hash: &str, failure: &GatewayRuntimeReloadError) -> Self { + Self { + config_revision, + policy_hash: policy_hash.to_string(), + failure_class: failure.class(), + } + } +} + async fn reload_gateway_policy_runtime( engine: &OpaEngine, policy: Option<&openshell_core::proto::SandboxPolicy>, @@ -2875,7 +2907,7 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { openshell_core::proto::EffectiveSetting, > = std::collections::HashMap::new(); let reloads_gateway_policy = ctx.loaded_policy_origin.allows_gateway_policy_reload(); - let mut last_failed_runtime_revision: Option<(u64, String)> = None; + let mut last_failed_runtime_revision: Option = None; let mut rejected_policy_generation: Option = None; let mut has_last_valid_policy = ctx.loaded_policy_origin.has_last_valid_policy(); @@ -3202,7 +3234,11 @@ async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { last_failed_runtime_revision = None; } Err(failure) => { - let failed_revision = (result.config_revision, result.policy_hash.clone()); + let failed_revision = FailedRuntimeRevision::new( + result.config_revision, + &result.policy_hash, + &failure, + ); if last_failed_runtime_revision.as_ref() != Some(&failed_revision) { let failure_mode = result.policy_validation_failure_mode; match apply_gateway_runtime_reload_failure( @@ -3856,6 +3892,56 @@ filesystem_policy: assert!(engine.fail_closed_reason().is_none()); } + #[test] + fn policy_rejection_after_middleware_outage_is_not_deduplicated() { + let engine = OpaEngine::from_strings( + include_str!("../../openshell-supervisor-network/data/sandbox-policy.rego"), + "network_policies: {}\n", + ) + .unwrap(); + let middleware_failure = GatewayRuntimeReloadError::MiddlewareRegistry(miette::miette!( + "middleware service unavailable" + )); + let first_failure = FailedRuntimeRevision::new(42, "sha256:candidate", &middleware_failure); + let middleware_disposition = apply_gateway_runtime_reload_failure( + &engine, + middleware_failure, + PolicyValidationFailureMode::FailClosed, + true, + 7, + ) + .unwrap(); + + assert!(matches!( + middleware_disposition, + GatewayRuntimeFailureDisposition::MiddlewareUnavailable { .. } + )); + assert!(engine.fail_closed_reason().is_none()); + + let policy_failure = GatewayRuntimeReloadError::PolicyValidation(miette::miette!( + "conflicting endpoint metadata" + )); + let second_failure = FailedRuntimeRevision::new(42, "sha256:candidate", &policy_failure); + assert_ne!( + first_failure, second_failure, + "a changed failure class for the same candidate must be handled" + ); + + let policy_disposition = apply_gateway_runtime_reload_failure( + &engine, + policy_failure, + PolicyValidationFailureMode::FailClosed, + true, + 7, + ) + .unwrap(); + assert!(matches!( + policy_disposition, + GatewayRuntimeFailureDisposition::PolicyRejected { .. } + )); + assert!(engine.fail_closed_reason().is_some()); + } + #[test] fn failed_gateway_runtime_snapshot_is_retried_without_revision_change() { let services = Vec::new();