diff --git a/Cargo.lock b/Cargo.lock index bfa3ab38..9ccbe632 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1377,6 +1377,31 @@ version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" +[[package]] +name = "computer-use-mcp" +version = "0.1.0" +dependencies = [ + "axum", + "base64", + "block2 0.6.2", + "core-foundation 0.10.0", + "core-graphics 0.24.0", + "log", + "objc2 0.6.4", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-foundation 0.3.2", + "objc2-image-io", + "objc2-screen-capture-kit", + "rmcp", + "schemars", + "serde", + "serde_json", + "tcode-services", + "tokio", + "uuid", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -5168,6 +5193,16 @@ dependencies = [ "objc2-quartz-core 0.3.2", ] +[[package]] +name = "objc2-av-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478ae33fcac9df0a18db8302387c666b8ef08a3e2d62b510ca4fc278a384b6c0" +dependencies = [ + "objc2 0.6.4", + "objc2-foundation 0.3.2", +] + [[package]] name = "objc2-cloud-kit" version = "0.3.2" @@ -5179,6 +5214,28 @@ dependencies = [ "objc2-foundation 0.3.2", ] +[[package]] +name = "objc2-core-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1eebcea8b0dbff5f7c8504f3107c68fc061a3eb44932051c8cf8a68d969c3b2" +dependencies = [ + "dispatch2", + "objc2 0.6.4", + "objc2-core-audio-types", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-core-audio-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a89f2ec274a0cf4a32642b2991e8b351a404d290da87bb6a9a9d8632490bd1c" +dependencies = [ + "bitflags 2.13.0", + "objc2 0.6.4", +] + [[package]] name = "objc2-core-data" version = "0.2.2" @@ -5209,7 +5266,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ "bitflags 2.13.0", + "block2 0.6.2", "dispatch2", + "libc", "objc2 0.6.4", ] @@ -5220,10 +5279,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e022c9d066895efa1345f8e33e584b9f958da2fd4cd116792e15e07e4720a807" dependencies = [ "bitflags 2.13.0", + "block2 0.6.2", "dispatch2", + "libc", "objc2 0.6.4", "objc2-core-foundation", "objc2-io-surface", + "objc2-metal 0.3.2", ] [[package]] @@ -5248,6 +5310,21 @@ dependencies = [ "objc2-foundation 0.3.2", ] +[[package]] +name = "objc2-core-media" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ec576860167a15dd9fce7fbee7512beb4e31f532159d3482d1f9c6caedf31d" +dependencies = [ + "bitflags 2.13.0", + "dispatch2", + "objc2 0.6.4", + "objc2-core-audio", + "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-core-video", +] + [[package]] name = "objc2-core-text" version = "0.3.2" @@ -5313,6 +5390,18 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "objc2-image-io" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b0446e98cf4a784cc7a0177715ff317eeaa8463841c616cfc78aa4f953c4ea" +dependencies = [ + "block2 0.6.2", + "objc2 0.6.4", + "objc2-core-foundation", + "objc2-core-graphics", +] + [[package]] name = "objc2-io-surface" version = "0.3.2" @@ -5324,6 +5413,16 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "objc2-javascript-core" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a1e6550c4caed348956ce3370c9ffeca70bb1dbed4fa96112e7c6170e074586" +dependencies = [ + "objc2 0.6.4", + "objc2-core-foundation", +] + [[package]] name = "objc2-metal" version = "0.2.2" @@ -5374,6 +5473,36 @@ dependencies = [ "objc2-metal 0.3.2", ] +[[package]] +name = "objc2-screen-capture-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74b7c5390f477482f001bc354d6571a70db7e4f8d5288e860c45521fbce11394" +dependencies = [ + "bitflags 2.13.0", + "block2 0.6.2", + "dispatch2", + "libc", + "objc2 0.6.4", + "objc2-av-foundation", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-core-media", + "objc2-foundation 0.3.2", + "objc2-uniform-type-identifiers", +] + +[[package]] +name = "objc2-security" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe137109bd1e8b5a99390f77a7d8b2961dafc1a1c5db8f2e60329ad6d895a" +dependencies = [ + "bitflags 2.13.0", + "objc2 0.6.4", + "objc2-core-foundation", +] + [[package]] name = "objc2-ui-kit" version = "0.3.2" @@ -5386,6 +5515,16 @@ dependencies = [ "objc2-foundation 0.3.2", ] +[[package]] +name = "objc2-uniform-type-identifiers" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7902ac02859fc1f7045f8b598c63f1ae0cc7efeaa06a9bc9f3d9a3c955974fa4" +dependencies = [ + "objc2 0.6.4", + "objc2-foundation 0.3.2", +] + [[package]] name = "objc2-web-kit" version = "0.3.2" @@ -5398,6 +5537,8 @@ dependencies = [ "objc2-app-kit 0.3.2", "objc2-core-foundation", "objc2-foundation 0.3.2", + "objc2-javascript-core", + "objc2-security", ] [[package]] @@ -8106,6 +8247,7 @@ name = "tcode" version = "0.1.0" dependencies = [ "agent", + "computer-use-mcp", "embed-resource", "env_logger", "gpui", @@ -8145,6 +8287,7 @@ version = "0.1.0" dependencies = [ "agent", "async-channel", + "computer-use-mcp", "gpui", "log", "orchestrate-mcp", @@ -8188,7 +8331,9 @@ dependencies = [ "agent", "async-channel", "base64", + "block2 0.6.2", "chrono", + "computer-use-mcp", "criterion", "gpui", "gpui-component", @@ -8196,6 +8341,12 @@ dependencies = [ "gpui-wry", "lb-wry", "log", + "objc2-app-kit 0.3.2", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-foundation 0.3.2", + "objc2-image-io", + "objc2-web-kit", "preview-mcp", "raw-window-handle", "rushdown", @@ -8426,6 +8577,7 @@ dependencies = [ "libc", "mio", "pin-project-lite", + "signal-hook-registry", "socket2", "tokio-macros", "windows-sys 0.61.2", diff --git a/Cargo.toml b/Cargo.toml index bb0e68ad..017e7eb5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "crates/term", "crates/preview-mcp", "crates/orchestrate-mcp", + "crates/computer-use-mcp", "crates/ui", "crates/app", ] diff --git a/crates/agent/examples/acp_probe.rs b/crates/agent/examples/acp_probe.rs index 664d97af..f09b4842 100644 --- a/crates/agent/examples/acp_probe.rs +++ b/crates/agent/examples/acp_probe.rs @@ -39,6 +39,7 @@ fn main() { interaction_mode: InteractionMode::Build, mcp_server: None, orchestrate_server: None, + computer_use_server: None, launch_env: LaunchEnv::default(), extra_args: Vec::new(), acp: Some(AcpAgent { diff --git a/crates/agent/examples/image_probe.rs b/crates/agent/examples/image_probe.rs index 81d413d0..c6fbce94 100644 --- a/crates/agent/examples/image_probe.rs +++ b/crates/agent/examples/image_probe.rs @@ -122,6 +122,7 @@ fn main() { interaction_mode: InteractionMode::Build, mcp_server: None, orchestrate_server: None, + computer_use_server: None, launch_env: Default::default(), extra_args: Vec::new(), acp: None, diff --git a/crates/agent/examples/interrupt_probe.rs b/crates/agent/examples/interrupt_probe.rs index 2a6c1aba..6fc6b08a 100644 --- a/crates/agent/examples/interrupt_probe.rs +++ b/crates/agent/examples/interrupt_probe.rs @@ -34,6 +34,7 @@ fn main() { interaction_mode: Default::default(), mcp_server: None, orchestrate_server: None, + computer_use_server: None, launch_env: Default::default(), extra_args: Vec::new(), acp: None, diff --git a/crates/agent/examples/probe.rs b/crates/agent/examples/probe.rs index 27e204b8..9796c9be 100644 --- a/crates/agent/examples/probe.rs +++ b/crates/agent/examples/probe.rs @@ -135,6 +135,7 @@ fn main() { interaction_mode, mcp_server: None, orchestrate_server: None, + computer_use_server: None, launch_env: Default::default(), extra_args: Vec::new(), acp: None, diff --git a/crates/agent/examples/read_only_probe.rs b/crates/agent/examples/read_only_probe.rs index 5f2cd57f..71797d4a 100644 --- a/crates/agent/examples/read_only_probe.rs +++ b/crates/agent/examples/read_only_probe.rs @@ -58,6 +58,7 @@ async fn run(provider: ProviderKind) -> i32 { interaction_mode: InteractionMode::Build, mcp_server: None, orchestrate_server: None, + computer_use_server: None, launch_env: Default::default(), extra_args: Vec::new(), acp: None, diff --git a/crates/agent/examples/steer_probe.rs b/crates/agent/examples/steer_probe.rs index 7d83a298..e94eed62 100644 --- a/crates/agent/examples/steer_probe.rs +++ b/crates/agent/examples/steer_probe.rs @@ -63,6 +63,7 @@ fn main() { interaction_mode: InteractionMode::Build, mcp_server: None, orchestrate_server: None, + computer_use_server: None, launch_env: Default::default(), extra_args: Vec::new(), acp: None, diff --git a/crates/agent/src/acp.rs b/crates/agent/src/acp.rs index 37ba111c..4eb75f36 100644 --- a/crates/agent/src/acp.rs +++ b/crates/agent/src/acp.rs @@ -410,10 +410,14 @@ async fn handshake( let caps = init.agent_capabilities.clone(); // Capability gate: tcode's MCP servers are loopback streamable-HTTP // endpoints, so they may only be offered to agents that speak MCP over HTTP. - let registrations: Vec<_> = [&opts.mcp_server, &opts.orchestrate_server] - .into_iter() - .flatten() - .collect(); + let registrations: Vec<_> = [ + &opts.mcp_server, + &opts.orchestrate_server, + &opts.computer_use_server, + ] + .into_iter() + .flatten() + .collect(); let mcp_servers = mcp_servers(®istrations, &caps); if !registrations.is_empty() && mcp_servers.is_empty() { log::info!( @@ -2261,6 +2265,17 @@ mod tests { }; assert_eq!(mcp_servers(&[®istration, &orchestrate], &caps).len(), 2); assert_eq!(mcp_servers(&[&orchestrate], &caps).len(), 1); + + let computer_use = McpRegistration { + name: McpRegistration::SERVER_NAME_COMPUTER_USE.into(), + url: "http://127.0.0.1:5322/mcp".into(), + bearer_token: "computer-token".into(), + }; + let servers = mcp_servers(&[®istration, &orchestrate, &computer_use], &caps); + assert_eq!(servers.len(), 3); + let value = serde_json::to_value(&servers[2]).unwrap(); + assert_eq!(value["name"], "tcode_computer_use"); + assert_eq!(value["headers"][0]["value"], "Bearer computer-token"); } #[test] diff --git a/crates/agent/src/claude.rs b/crates/agent/src/claude.rs index 0b59f312..2981233c 100644 --- a/crates/agent/src/claude.rs +++ b/crates/agent/src/claude.rs @@ -114,7 +114,11 @@ pub async fn start(opts: SessionOptions) -> Result { } // Register tcode's enabled HTTP MCP servers. Tokens ride in Authorization // headers inside the merged `--mcp-config` JSON. - for arg in mcp_args(opts.mcp_server.as_ref(), opts.orchestrate_server.as_ref()) { + for arg in mcp_args( + opts.mcp_server.as_ref(), + opts.orchestrate_server.as_ref(), + opts.computer_use_server.as_ref(), + ) { cmd.arg(arg); } // Settings → Providers "Launch arguments", appended last so the user can @@ -255,8 +259,12 @@ pub async fn start(opts: SessionOptions) -> Result { fn mcp_args( preview: Option<&crate::McpRegistration>, orchestrate: Option<&crate::McpRegistration>, + computer_use: Option<&crate::McpRegistration>, ) -> Vec { - let registrations: Vec<_> = [preview, orchestrate].into_iter().flatten().collect(); + let registrations: Vec<_> = [preview, orchestrate, computer_use] + .into_iter() + .flatten() + .collect(); if registrations.is_empty() { Vec::new() } else { @@ -2868,16 +2876,24 @@ mod tests { url: "http://o".into(), bearer_token: "o".into(), }; - assert!(mcp_args(None, None).is_empty()); - let one = mcp_args(Some(&preview), None); + let computer_use = crate::McpRegistration { + name: "tcode_computer_use".into(), + url: "http://c".into(), + bearer_token: "c".into(), + }; + assert!(mcp_args(None, None, None).is_empty()); + let one = mcp_args(Some(&preview), None, None); assert_eq!(one[0], "--mcp-config"); let one_json: Value = serde_json::from_str(&one[1]).unwrap(); assert!(one_json["mcpServers"].get("tcode_preview").is_some()); assert!(one_json["mcpServers"].get("tcode_orchestrate").is_none()); - let both_json: Value = - serde_json::from_str(&mcp_args(Some(&preview), Some(&orchestrate))[1]).unwrap(); - assert!(both_json["mcpServers"].get("tcode_preview").is_some()); - assert!(both_json["mcpServers"].get("tcode_orchestrate").is_some()); + let all_json: Value = serde_json::from_str( + &mcp_args(Some(&preview), Some(&orchestrate), Some(&computer_use))[1], + ) + .unwrap(); + assert!(all_json["mcpServers"].get("tcode_preview").is_some()); + assert!(all_json["mcpServers"].get("tcode_orchestrate").is_some()); + assert!(all_json["mcpServers"].get("tcode_computer_use").is_some()); } fn feed(mapper: &mut Mapper, line: &str) -> Vec { diff --git a/crates/agent/src/codex.rs b/crates/agent/src/codex.rs index 50fecdd4..e90b0601 100644 --- a/crates/agent/src/codex.rs +++ b/crates/agent/src/codex.rs @@ -405,7 +405,11 @@ async fn run_actor( ready: Sender>, ) { // Register tcode's enabled streamable-HTTP MCP servers via `-c` overrides. - let mut extra_args = mcp_args(opts.mcp_server.as_ref(), opts.orchestrate_server.as_ref()); + let mut extra_args = mcp_args( + opts.mcp_server.as_ref(), + opts.orchestrate_server.as_ref(), + opts.computer_use_server.as_ref(), + ); // Any additional launch arguments configured for this provider. extra_args.extend(opts.extra_args.iter().cloned()); let (mut child, mut stdin, lines, mut stderr_tail) = @@ -538,8 +542,9 @@ async fn run_actor( fn mcp_args( preview: Option<&crate::McpRegistration>, orchestrate: Option<&crate::McpRegistration>, + computer_use: Option<&crate::McpRegistration>, ) -> Vec { - [preview, orchestrate] + [preview, orchestrate, computer_use] .into_iter() .flatten() .flat_map(|mcp| ["-c".to_string(), mcp.codex_config_override()]) @@ -2041,14 +2046,20 @@ mod tests { url: "http://o".into(), bearer_token: "o".into(), }; - assert!(mcp_args(None, None).is_empty()); - let one = mcp_args(Some(&preview), None); + let computer_use = crate::McpRegistration { + name: "tcode_computer_use".into(), + url: "http://c".into(), + bearer_token: "c".into(), + }; + assert!(mcp_args(None, None, None).is_empty()); + let one = mcp_args(Some(&preview), None, None); assert_eq!(one.len(), 2); assert!(one[1].starts_with("mcp_servers.tcode_preview=")); - let both = mcp_args(Some(&preview), Some(&orchestrate)); - assert_eq!(both.len(), 4); - assert!(both[1].starts_with("mcp_servers.tcode_preview=")); - assert!(both[3].starts_with("mcp_servers.tcode_orchestrate=")); + let all = mcp_args(Some(&preview), Some(&orchestrate), Some(&computer_use)); + assert_eq!(all.len(), 6); + assert!(all[1].starts_with("mcp_servers.tcode_preview=")); + assert!(all[3].starts_with("mcp_servers.tcode_orchestrate=")); + assert!(all[5].starts_with("mcp_servers.tcode_computer_use=")); } fn test_actor() -> (Actor, Receiver) { diff --git a/crates/agent/src/lib.rs b/crates/agent/src/lib.rs index ccee46cd..cb63702f 100644 --- a/crates/agent/src/lib.rs +++ b/crates/agent/src/lib.rs @@ -127,6 +127,9 @@ pub struct SessionOptions { /// The tcode orchestrator MCP server, scoped to this parent session by its /// bearer token. Only orchestrate-enabled sessions receive it. pub orchestrate_server: Option, + /// The process-wide computer-use MCP server. The runtime supplies it only + /// when the global computer-use setting is enabled. + pub computer_use_server: Option, /// Per-provider environment (Settings → Providers): extra variables merged /// into the child's environment, plus the home-directory override. See /// [`LaunchEnv`]. @@ -189,6 +192,7 @@ pub struct McpRegistration { impl McpRegistration { pub const SERVER_NAME_PREVIEW: &'static str = "tcode_preview"; pub const SERVER_NAME_ORCHESTRATE: &'static str = "tcode_orchestrate"; + pub const SERVER_NAME_COMPUTER_USE: &'static str = "tcode_computer_use"; /// Claude Code `--mcp-config` JSON: a single `mcpServers` map entry for an /// HTTP server carrying the bearer token as an `Authorization` header. diff --git a/crates/app/Cargo.toml b/crates/app/Cargo.toml index 889b84b7..763a31d2 100644 --- a/crates/app/Cargo.toml +++ b/crates/app/Cargo.toml @@ -20,6 +20,7 @@ tcode-runtime = { path = "../runtime" } tcode-ui = { path = "../ui" } preview-mcp = { path = "../preview-mcp" } orchestrate-mcp = { path = "../orchestrate-mcp" } +computer-use-mcp = { path = "../computer-use-mcp" } gpui = { git = "https://github.com/zed-industries/zed" } gpui_platform = { git = "https://github.com/zed-industries/zed", features = ["font-kit", "x11", "wayland"] } gpui-component = { git = "https://github.com/longbridge/gpui-component" } diff --git a/crates/app/src/main.rs b/crates/app/src/main.rs index 1b60d7c5..67e5ecfd 100644 --- a/crates/app/src/main.rs +++ b/crates/app/src/main.rs @@ -327,6 +327,15 @@ fn main() { } Err(err) => log::warn!("orchestrate MCP server failed to start: {err}"), } + match computer_use_mcp::start() { + Ok(server) => { + log::info!("computer-use MCP server listening at {}", server.url); + app_state.update(cx, |state, _| { + state.attach_computer_use_mcp(server.url, server.token) + }); + } + Err(err) => log::warn!("computer-use MCP server failed to start: {err}"), + } // Refresh the model catalogs in the background so the picker shows // real, up-to-date models (the persisted cache serves until then). app_state.update(cx, |state, cx| state.refresh_model_catalogs(cx)); @@ -362,6 +371,11 @@ fn main() { state.debug_provider_expanded = dexp; }); } + // Restart continuity: if this launch follows a permission-grant + // relaunch, reopen the recorded session and Settings page. Runs + // synchronously before the window (and settings page) is built, so + // the page mounts already on the recorded section. No-op otherwise. + app_state.update(cx, |state, cx| state.apply_pending_relaunch(cx)); let debug_seed = debug_compose.is_some() || debug_image.is_some() || debug_cwd.is_some() diff --git a/crates/computer-use-mcp/Cargo.toml b/crates/computer-use-mcp/Cargo.toml new file mode 100644 index 00000000..4719ec98 --- /dev/null +++ b/crates/computer-use-mcp/Cargo.toml @@ -0,0 +1,45 @@ +[package] +name = "computer-use-mcp" +version = "0.1.0" +edition = "2024" + +[dependencies] +rmcp = { version = "0.15", features = [ + "macros", + "schemars", + "server", + "transport-streamable-http-server", +] } +axum = "0.8" +tokio = { version = "1", features = [ + "rt-multi-thread", + "net", + "time", + "sync", + "macros", + "process", +] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +schemars = "1" +uuid = { version = "1", features = ["v4"] } +log = "0.4" +base64 = "0.22" +tcode-services = { path = "../services" } + +[target.'cfg(target_os = "macos")'.dependencies] +block2 = "0.6.2" +core-foundation = "0.10" +core-graphics = { version = "0.24", features = ["highsierra"] } +objc2 = "0.6.4" +objc2-core-foundation = { version = "0.3.2", features = ["CFData", "CFString"] } +objc2-core-graphics = { version = "0.3.2", features = ["CGImage"] } +objc2-foundation = { version = "0.3.2", features = ["NSArray", "NSError"] } +objc2-image-io = { version = "0.3.2", features = ["CGImageDestination"] } +objc2-screen-capture-kit = { version = "0.3.2", features = [ + "SCScreenshotManager", + "SCShareableContent", +] } + +[dev-dependencies] +tokio = { version = "1", features = ["rt", "macros", "time"] } diff --git a/crates/computer-use-mcp/src/backend.rs b/crates/computer-use-mcp/src/backend.rs new file mode 100644 index 00000000..8f433e19 --- /dev/null +++ b/crates/computer-use-mcp/src/backend.rs @@ -0,0 +1,389 @@ +//! Platform backend contract and platform-neutral input descriptions. + +use std::fmt; + +use serde::{Deserialize, Serialize}; + +use crate::outline::{Frame, UiNode}; + +#[cfg(target_os = "macos")] +mod macos; +#[cfg(not(target_os = "macos"))] +mod stub; + +const UNSUPPORTED_MESSAGE: &str = "computer use is unsupported on this platform"; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RootKind { + #[default] + Window, + Dialog, + Sheet, + Menu, + Popover, +} + +impl fmt::Display for RootKind { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(match self { + Self::Window => "window", + Self::Dialog => "dialog", + Self::Sheet => "sheet", + Self::Menu => "menu", + Self::Popover => "popover", + }) + } +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct RootInfo { + pub ref_id: String, + pub app_name: String, + pub bundle_id: String, + pub pid: u32, + pub title: String, + pub kind: RootKind, + pub window_id: u32, + pub frame: Frame, +} + +impl RootInfo { + pub fn identity(&self) -> String { + format!("{}:{}", self.pid, self.window_id) + } +} + +#[derive(Debug, Clone, Default)] +pub struct RootFilters { + pub text: Option, + pub app: Option, + pub bundle_id: Option, + pub pid: Option, + pub kind: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CapturePolicy { + Never, + Always, + IfSparse, +} + +#[derive(Debug, Clone, Copy)] +pub struct ObserveRequest { + pub semantic: bool, + pub capture: CapturePolicy, +} + +#[derive(Debug, Clone)] +pub struct RootObservation { + pub root: RootInfo, + pub tree: UiNode, + pub screenshot_png: Option>, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ActionKind { + Press, + Click, + SetText, + TypeText, + Keypress, + Scroll, + Drag, + MoveMouse, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum MouseButton { + #[default] + Left, + Right, + Middle, +} + +#[derive(Debug, Clone)] +pub struct ActionRequest { + pub kind: ActionKind, + pub target_path: Option>, + pub target_frame: Option, + pub target_role: Option, + pub target_title: Option, + pub target_actions: Vec, + pub x: Option, + pub y: Option, + pub text: Option, + pub keys: Option>, + pub scroll_x: Option, + pub scroll_y: Option, + pub path: Option>, + pub button: MouseButton, + pub click_count: u32, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ActionOutcome { + Worked, + Didnt, + Unknown, +} + +#[derive(Debug, Clone, Serialize)] +pub struct ActionResult { + pub outcome: ActionOutcome, + pub message: String, +} + +impl ActionResult { + pub fn worked(message: impl Into) -> Self { + Self { + outcome: ActionOutcome::Worked, + message: message.into(), + } + } + + pub fn didnt(message: impl Into) -> Self { + Self { + outcome: ActionOutcome::Didnt, + message: message.into(), + } + } + + pub fn unknown(message: impl Into) -> Self { + Self { + outcome: ActionOutcome::Unknown, + message: message.into(), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum BackendErrorCode { + UnsupportedPlatform, + RootNotFound, + ObservationFailed, + CaptureFailed, + InvalidAction, + OperationFailed, +} + +#[derive(Debug, Clone, Serialize)] +pub struct BackendError { + pub code: BackendErrorCode, + pub message: String, +} + +impl BackendError { + pub fn new(code: BackendErrorCode, message: impl Into) -> Self { + Self { + code, + message: message.into(), + } + } + + pub fn unsupported() -> Self { + Self::new(BackendErrorCode::UnsupportedPlatform, UNSUPPORTED_MESSAGE) + } +} + +impl fmt::Display for BackendError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "{}", self.message) + } +} + +impl std::error::Error for BackendError {} + +pub trait Backend: Send + Sync { + fn list_roots(&self, filters: &RootFilters) -> Result, BackendError>; + + fn observe( + &self, + root: &RootInfo, + request: ObserveRequest, + ) -> Result; + + fn perform_action( + &self, + root: &RootInfo, + request: &ActionRequest, + ) -> Result; + + fn read_element_text( + &self, + root: &RootInfo, + target_path: &[usize], + ) -> Result; +} + +pub fn platform_backend() -> Box { + #[cfg(target_os = "macos")] + { + Box::new(macos::MacosBackend) + } + #[cfg(not(target_os = "macos"))] + { + Box::new(stub::StubBackend) + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct KeyModifiers { + pub command: bool, + pub control: bool, + pub option: bool, + pub shift: bool, + pub function: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct KeyChord { + pub keycode: u16, + pub modifiers: KeyModifiers, +} + +pub fn parse_key_chord(keys: &[String]) -> Result { + let parts: Vec = keys + .iter() + .flat_map(|part| part.split('+')) + .map(|part| part.trim().to_ascii_lowercase()) + .filter(|part| !part.is_empty()) + .collect(); + if parts.is_empty() { + return Err("keypress requires a key name or chord".into()); + } + + let mut modifiers = KeyModifiers::default(); + let mut keycode = None; + for part in parts { + match part.as_str() { + "cmd" | "command" | "meta" => modifiers.command = true, + "ctrl" | "control" => modifiers.control = true, + "alt" | "option" => modifiers.option = true, + "shift" => modifiers.shift = true, + "fn" | "function" => modifiers.function = true, + key => { + if keycode.is_some() { + return Err("keypress accepts exactly one non-modifier key".into()); + } + keycode = keycode_for_name(key); + if keycode.is_none() { + return Err(format!("unknown key name: {key}")); + } + } + } + } + let keycode = keycode.ok_or_else(|| "keypress chord has no non-modifier key".to_string())?; + Ok(KeyChord { keycode, modifiers }) +} + +/// US ANSI virtual key codes. Key names are layout-independent controls or +/// physical letter/number keys; text entry uses Unicode events instead. +pub fn keycode_for_name(name: &str) -> Option { + Some(match name.trim().to_ascii_lowercase().as_str() { + "a" => 0x00, + "s" => 0x01, + "d" => 0x02, + "f" => 0x03, + "h" => 0x04, + "g" => 0x05, + "z" => 0x06, + "x" => 0x07, + "c" => 0x08, + "v" => 0x09, + "b" => 0x0B, + "q" => 0x0C, + "w" => 0x0D, + "e" => 0x0E, + "r" => 0x0F, + "y" => 0x10, + "t" => 0x11, + "1" => 0x12, + "2" => 0x13, + "3" => 0x14, + "4" => 0x15, + "6" => 0x16, + "5" => 0x17, + "=" | "equal" => 0x18, + "9" => 0x19, + "7" => 0x1A, + "-" | "minus" => 0x1B, + "8" => 0x1C, + "0" => 0x1D, + "]" | "right_bracket" => 0x1E, + "o" => 0x1F, + "u" => 0x20, + "[" | "left_bracket" => 0x21, + "i" => 0x22, + "p" => 0x23, + "enter" | "return" => 0x24, + "l" => 0x25, + "j" => 0x26, + "'" | "quote" => 0x27, + "k" => 0x28, + ";" | "semicolon" => 0x29, + "\\" | "backslash" => 0x2A, + "," | "comma" => 0x2B, + "/" | "slash" => 0x2C, + "n" => 0x2D, + "m" => 0x2E, + "." | "period" => 0x2F, + "tab" => 0x30, + "space" => 0x31, + "`" | "grave" => 0x32, + "delete" | "backspace" => 0x33, + "escape" | "esc" => 0x35, + "f17" => 0x40, + "f18" => 0x4F, + "f19" => 0x50, + "f20" => 0x5A, + "f5" => 0x60, + "f6" => 0x61, + "f7" => 0x62, + "f3" => 0x63, + "f8" => 0x64, + "f9" => 0x65, + "f11" => 0x67, + "f13" => 0x69, + "f16" => 0x6A, + "f14" => 0x6B, + "f10" => 0x6D, + "f12" => 0x6F, + "f15" => 0x71, + "help" | "insert" => 0x72, + "home" => 0x73, + "page_up" | "pageup" => 0x74, + "forward_delete" => 0x75, + "f4" => 0x76, + "end" => 0x77, + "f2" => 0x78, + "page_down" | "pagedown" => 0x79, + "f1" => 0x7A, + "left" | "left_arrow" => 0x7B, + "right" | "right_arrow" => 0x7C, + "down" | "down_arrow" => 0x7D, + "up" | "up_arrow" => 0x7E, + _ => return None, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn key_names_and_chords_map_to_macos_virtual_codes() { + assert_eq!(keycode_for_name("enter"), Some(0x24)); + assert_eq!(keycode_for_name("left_arrow"), Some(0x7B)); + assert_eq!(keycode_for_name("F12"), Some(0x6F)); + assert_eq!(keycode_for_name("definitely-not-a-key"), None); + + let chord = parse_key_chord(&["cmd+shift+s".into()]).unwrap(); + assert_eq!(chord.keycode, 0x01); + assert!(chord.modifiers.command); + assert!(chord.modifiers.shift); + } +} diff --git a/crates/computer-use-mcp/src/backend/macos/ax.rs b/crates/computer-use-mcp/src/backend/macos/ax.rs new file mode 100644 index 00000000..426d85ed --- /dev/null +++ b/crates/computer-use-mcp/src/backend/macos/ax.rs @@ -0,0 +1,683 @@ +use std::collections::HashSet; +use std::ffi::c_void; +use std::fmt; +use std::ptr; + +use core_foundation::array::{CFArrayGetCount, CFArrayGetValueAtIndex, CFArrayRef}; +use core_foundation::base::{CFGetTypeID, CFRelease, CFRetain, CFTypeID, CFTypeRef, TCFType}; +use core_foundation::boolean::CFBoolean; +use core_foundation::number::CFNumber; +use core_foundation::string::{CFString, CFStringRef}; +use core_graphics::geometry::{CGPoint, CGRect, CGSize}; + +use super::super::{BackendError, BackendErrorCode, RootInfo, RootKind}; +use crate::outline::{Frame, UiNode, canonical_role}; + +type AXUIElementRef = CFTypeRef; +type AXValueRef = CFTypeRef; +type AXError = i32; +type AXValueType = u32; + +const AX_SUCCESS: AXError = 0; +const AX_VALUE_CGPOINT: AXValueType = 1; +const AX_VALUE_CGSIZE: AXValueType = 2; +const AX_VALUE_CGRECT: AXValueType = 3; +const MAX_DEPTH: usize = 18; +const MAX_NODES: usize = 3_000; +const MAX_CHILDREN_PER_NODE: usize = 500; + +#[link(name = "ApplicationServices", kind = "framework")] +unsafe extern "C" { + fn AXUIElementCreateApplication(pid: i32) -> AXUIElementRef; + fn AXUIElementGetTypeID() -> CFTypeID; + fn AXUIElementCopyAttributeValue( + element: AXUIElementRef, + attribute: CFStringRef, + value: *mut CFTypeRef, + ) -> AXError; + fn AXUIElementGetAttributeValueCount( + element: AXUIElementRef, + attribute: CFStringRef, + count: *mut isize, + ) -> AXError; + fn AXUIElementCopyAttributeValues( + element: AXUIElementRef, + attribute: CFStringRef, + index: isize, + max_values: isize, + values: *mut CFArrayRef, + ) -> AXError; + fn AXUIElementCopyActionNames(element: AXUIElementRef, names: *mut CFArrayRef) -> AXError; + fn AXUIElementPerformAction(element: AXUIElementRef, action: CFStringRef) -> AXError; + fn AXUIElementSetAttributeValue( + element: AXUIElementRef, + attribute: CFStringRef, + value: CFTypeRef, + ) -> AXError; + fn AXValueGetTypeID() -> CFTypeID; + fn AXValueGetType(value: AXValueRef) -> AXValueType; + fn AXValueGetValue(value: AXValueRef, value_type: AXValueType, value_ptr: *mut c_void) -> bool; +} + +#[link(name = "AppKit", kind = "framework")] +unsafe extern "C" {} + +#[link(name = "objc")] +unsafe extern "C" { + fn objc_getClass(name: *const std::ffi::c_char) -> *mut c_void; + fn sel_registerName(name: *const std::ffi::c_char) -> *mut c_void; + fn objc_msgSend(); +} + +struct OwnedCf(CFTypeRef); + +impl OwnedCf { + fn as_ax(&self) -> AXUIElementRef { + self.0 + } + + unsafe fn from_create(value: CFTypeRef) -> Option { + (!value.is_null()).then_some(Self(value)) + } + + unsafe fn from_borrowed(value: CFTypeRef) -> Option { + if value.is_null() { + None + } else { + // SAFETY: the caller supplied a live CF object borrowed from a + // container; retaining gives this wrapper independent ownership. + unsafe { CFRetain(value) }; + Some(Self(value)) + } + } +} + +impl Drop for OwnedCf { + fn drop(&mut self) { + // SAFETY: OwnedCf is constructed only for create-rule or retained refs. + unsafe { CFRelease(self.0) }; + } +} + +#[derive(Debug)] +pub(super) struct AxFailure { + operation: &'static str, + code: AXError, +} + +impl fmt::Display for AxFailure { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "{} failed with AX error {} ({})", + self.operation, + self.code, + ax_error_name(self.code) + ) + } +} + +pub(super) struct Target { + _application: OwnedCf, + element: OwnedCf, +} + +impl Target { + pub(super) fn press(&self) -> Result<(), AxFailure> { + let action = CFString::new("AXPress"); + let code = + unsafe { AXUIElementPerformAction(self.element.as_ax(), action.as_concrete_TypeRef()) }; + ax_result("AXPress", code) + } + + pub(super) fn set_text(&self, text: &str) -> Result<(), AxFailure> { + let attribute = CFString::new("AXValue"); + let value = CFString::new(text); + let code = unsafe { + AXUIElementSetAttributeValue( + self.element.as_ax(), + attribute.as_concrete_TypeRef(), + value.as_CFTypeRef(), + ) + }; + ax_result("setting AXValue", code) + } + + pub(super) fn focus(&self) -> Result<(), AxFailure> { + let attribute = CFString::new("AXFocused"); + let code = unsafe { + AXUIElementSetAttributeValue( + self.element.as_ax(), + attribute.as_concrete_TypeRef(), + CFBoolean::true_value().as_CFTypeRef(), + ) + }; + ax_result("setting AXFocused", code) + } + + pub(super) fn frame(&self) -> Frame { + element_frame(self.element.as_ax()) + } + + pub(super) fn text(&self) -> String { + element_text(self.element.as_ax()) + } +} + +pub(super) fn application_identifier(pid: u32) -> String { + if let Some(identifier) = running_application_identifier(pid) { + return identifier; + } + let Some(application) = create_application(pid) else { + return String::new(); + }; + attribute_string(application.as_ax(), "AXIdentifier").unwrap_or_default() +} + +fn running_application_identifier(pid: u32) -> Option { + // SAFETY: all selectors and return types below are stable Foundation/AppKit + // APIs. The autorelease pool bounds temporary Objective-C objects created + // on the MCP runtime thread. + unsafe { + let pool_class = objc_getClass(c"NSAutoreleasePool".as_ptr()); + let application_class = objc_getClass(c"NSRunningApplication".as_ptr()); + if pool_class.is_null() || application_class.is_null() { + return None; + } + let pool = send_id(pool_class, c"new"); + let application = send_id_i32( + application_class, + c"runningApplicationWithProcessIdentifier:", + pid as i32, + ); + let result = if application.is_null() { + None + } else { + let identifier = send_id(application, c"bundleIdentifier"); + if identifier.is_null() { + None + } else { + let utf8 = send_id(identifier, c"UTF8String") as *const std::ffi::c_char; + (!utf8.is_null()).then(|| { + std::ffi::CStr::from_ptr(utf8) + .to_string_lossy() + .into_owned() + }) + } + }; + send_void(pool, c"drain"); + result + } +} + +unsafe fn send_id(receiver: *mut c_void, selector: &std::ffi::CStr) -> *mut c_void { + let send: unsafe extern "C" fn(*mut c_void, *mut c_void) -> *mut c_void = + unsafe { std::mem::transmute(objc_msgSend as unsafe extern "C" fn()) }; + unsafe { send(receiver, sel_registerName(selector.as_ptr())) } +} + +unsafe fn send_id_i32(receiver: *mut c_void, selector: &std::ffi::CStr, value: i32) -> *mut c_void { + let send: unsafe extern "C" fn(*mut c_void, *mut c_void, i32) -> *mut c_void = + unsafe { std::mem::transmute(objc_msgSend as unsafe extern "C" fn()) }; + unsafe { send(receiver, sel_registerName(selector.as_ptr()), value) } +} + +unsafe fn send_void(receiver: *mut c_void, selector: &std::ffi::CStr) { + let send: unsafe extern "C" fn(*mut c_void, *mut c_void) = + unsafe { std::mem::transmute(objc_msgSend as unsafe extern "C" fn()) }; + unsafe { send(receiver, sel_registerName(selector.as_ptr())) } +} + +pub(super) fn root_kind(root: &RootInfo) -> RootKind { + let Ok((_application, window)) = locate_window(root) else { + return RootKind::Window; + }; + let role = attribute_string(window.as_ax(), "AXRole").unwrap_or_default(); + let subrole = attribute_string(window.as_ax(), "AXSubrole").unwrap_or_default(); + let combined = format!("{role} {subrole}").to_ascii_lowercase(); + if combined.contains("sheet") { + RootKind::Sheet + } else if combined.contains("dialog") || combined.contains("systemdialog") { + RootKind::Dialog + } else if combined.contains("popover") { + RootKind::Popover + } else if combined.contains("menu") { + RootKind::Menu + } else { + RootKind::Window + } +} + +pub(super) fn observe_tree(root: &RootInfo) -> Result { + let (_application, window) = locate_window(root)?; + let mut context = WalkContext { + count: 0, + visited: HashSet::new(), + root_frame: root.frame, + }; + let mut tree = walk_element(window.as_ax(), 0, &mut context).ok_or_else(|| { + BackendError::new( + BackendErrorCode::ObservationFailed, + format!("the accessibility tree for {} was empty", root.title), + ) + })?; + if tree.title.is_empty() { + tree.title.clone_from(&root.title); + } + if !tree.frame.has_area() { + tree.frame = root.frame; + } + Ok(tree) +} + +pub(super) fn locate_target( + root: &RootInfo, + path: &[usize], + expected_role: Option<&str>, + expected_title: Option<&str>, +) -> Result { + let (application, mut element) = locate_window(root)?; + for &index in path { + let children = copy_children(element.as_ax(), index.saturating_add(1)); + let Some(child) = children.into_iter().nth(index) else { + return Err(BackendError::new( + BackendErrorCode::OperationFailed, + "the target's accessibility path moved; call observe_ui again", + )); + }; + element = child; + } + + if let Some(expected_role) = expected_role { + let actual = attribute_string(element.as_ax(), "AXRole").unwrap_or_default(); + if canonical_role(&actual) != canonical_role(expected_role) { + return Err(BackendError::new( + BackendErrorCode::OperationFailed, + format!( + "target role changed from {} to {}; call observe_ui again", + canonical_role(expected_role), + canonical_role(&actual) + ), + )); + } + } + if let Some(expected_title) = expected_title.filter(|title| !title.is_empty()) { + let actual = attribute_string(element.as_ax(), "AXTitle").unwrap_or_default(); + if !actual.is_empty() && actual != expected_title { + return Err(BackendError::new( + BackendErrorCode::OperationFailed, + "target title changed; call observe_ui again", + )); + } + } + Ok(Target { + _application: application, + element, + }) +} + +pub(super) fn read_target_text(root: &RootInfo, path: &[usize]) -> Result { + Ok(locate_target(root, path, None, None)?.text()) +} + +fn create_application(pid: u32) -> Option { + let application = unsafe { AXUIElementCreateApplication(pid as i32) }; + // SAFETY: AXUIElementCreateApplication follows the create rule. + unsafe { OwnedCf::from_create(application) } +} + +fn locate_window(root: &RootInfo) -> Result<(OwnedCf, OwnedCf), BackendError> { + let application = create_application(root.pid).ok_or_else(|| { + BackendError::new( + BackendErrorCode::RootNotFound, + format!("could not create an AX application for pid {}", root.pid), + ) + })?; + let windows = copy_attribute_elements(application.as_ax(), "AXWindows", 200); + let window = windows + .into_iter() + .max_by(|left, right| { + window_match_score(left.as_ax(), root) + .total_cmp(&window_match_score(right.as_ax(), root)) + }) + .ok_or_else(|| { + BackendError::new( + BackendErrorCode::RootNotFound, + format!("no AX window matched root {}", root.ref_id), + ) + })?; + Ok((application, window)) +} + +fn window_match_score(window: AXUIElementRef, root: &RootInfo) -> f64 { + let title = attribute_string(window, "AXTitle").unwrap_or_default(); + let frame = element_frame(window); + let exact_title = if !root.title.is_empty() && title == root.title { + 1_000_000.0 + } else if title.to_lowercase().contains(&root.title.to_lowercase()) { + 100_000.0 + } else { + 0.0 + }; + let frame_distance = (frame.x - root.frame.x).abs() + + (frame.y - root.frame.y).abs() + + (frame.w - root.frame.w).abs() + + (frame.h - root.frame.h).abs(); + exact_title - frame_distance +} + +struct WalkContext { + count: usize, + visited: HashSet, + root_frame: Frame, +} + +fn walk_element( + element: AXUIElementRef, + depth: usize, + context: &mut WalkContext, +) -> Option { + if context.count >= MAX_NODES || !context.visited.insert(element as usize) { + return None; + } + context.count += 1; + let role = attribute_string(element, "AXRole").unwrap_or_else(|| "AXUnknown".into()); + let title = attribute_string(element, "AXTitle") + .or_else(|| attribute_string(element, "AXLabel")) + .unwrap_or_default(); + let value = attribute_text(element, "AXValue").unwrap_or_default(); + let description = ["AXDescription", "AXHelp", "AXRoleDescription"] + .into_iter() + .filter_map(|attribute| attribute_string(element, attribute)) + .find(|description| !description.is_empty() && description != &title) + .unwrap_or_default(); + let frame = element_frame(element); + let actions = action_names(element); + let enabled = attribute_bool(element, "AXEnabled").unwrap_or(true); + let focused = attribute_bool(element, "AXFocused").unwrap_or(false); + let mut node = UiNode { + ref_id: String::new(), + role: canonical_role(&role), + title, + value, + description, + frame, + actions, + enabled, + focused, + children: Vec::new(), + }; + + if depth < MAX_DEPTH && context.count < MAX_NODES { + let remaining = MAX_NODES - context.count; + for child in copy_children(element, remaining.min(MAX_CHILDREN_PER_NODE)) { + let Some(child) = walk_element(child.as_ax(), depth + 1, context) else { + continue; + }; + let invisible_leaf = child.children.is_empty() + && !child.is_interactive() + && child.title.is_empty() + && child.value.is_empty() + && (!child.frame.has_area() + || (context.root_frame.has_area() + && !child.frame.intersects(context.root_frame))); + if !invisible_leaf { + node.children.push(child); + } + if context.count >= MAX_NODES { + break; + } + } + } + Some(node) +} + +fn copy_children(element: AXUIElementRef, maximum: usize) -> Vec { + copy_attribute_elements(element, "AXChildren", maximum) +} + +fn copy_attribute_elements( + element: AXUIElementRef, + attribute_name: &str, + maximum: usize, +) -> Vec { + if maximum == 0 { + return Vec::new(); + } + let attribute = CFString::new(attribute_name); + let mut count = 0_isize; + let count_code = unsafe { + AXUIElementGetAttributeValueCount(element, attribute.as_concrete_TypeRef(), &mut count) + }; + if count_code != AX_SUCCESS || count <= 0 { + return Vec::new(); + } + let count = usize::try_from(count).unwrap_or(0).min(maximum); + let mut array: CFArrayRef = ptr::null(); + let code = unsafe { + AXUIElementCopyAttributeValues( + element, + attribute.as_concrete_TypeRef(), + 0, + count as isize, + &mut array, + ) + }; + if code != AX_SUCCESS || array.is_null() { + return Vec::new(); + } + // SAFETY: CopyAttributeValues returns a create-rule CFArray. + let Some(array_owner) = (unsafe { OwnedCf::from_create(array as CFTypeRef) }) else { + return Vec::new(); + }; + let actual_count = unsafe { CFArrayGetCount(array) }.max(0) as usize; + let mut values = Vec::with_capacity(actual_count); + for index in 0..actual_count { + let value = unsafe { CFArrayGetValueAtIndex(array, index as isize) } as CFTypeRef; + if value.is_null() || unsafe { CFGetTypeID(value) } != unsafe { AXUIElementGetTypeID() } { + continue; + } + // SAFETY: value is borrowed from array_owner, which remains live. + if let Some(value) = unsafe { OwnedCf::from_borrowed(value) } { + values.push(value); + } + } + drop(array_owner); + values +} + +fn copy_attribute(element: AXUIElementRef, attribute_name: &str) -> Option { + let attribute = CFString::new(attribute_name); + let mut value: CFTypeRef = ptr::null(); + let code = unsafe { + AXUIElementCopyAttributeValue(element, attribute.as_concrete_TypeRef(), &mut value) + }; + if code == AX_SUCCESS { + // SAFETY: CopyAttributeValue returns a create-rule object on success. + unsafe { OwnedCf::from_create(value) } + } else { + None + } +} + +fn attribute_string(element: AXUIElementRef, attribute_name: &str) -> Option { + let value = copy_attribute(element, attribute_name)?; + cf_string(&value) +} + +fn attribute_text(element: AXUIElementRef, attribute_name: &str) -> Option { + let value = copy_attribute(element, attribute_name)?; + cf_string(&value).or_else(|| { + if unsafe { CFGetTypeID(value.0) } == CFNumber::type_id() { + // SAFETY: the type id was checked and wrapping under get retains it. + let number = unsafe { CFNumber::wrap_under_get_rule(value.0.cast()) }; + number + .to_i64() + .map(|number| number.to_string()) + .or_else(|| number.to_f64().map(|number| number.to_string())) + } else if unsafe { CFGetTypeID(value.0) } == CFBoolean::type_id() { + // SAFETY: the type id was checked and wrapping under get retains it. + let boolean = unsafe { CFBoolean::wrap_under_get_rule(value.0.cast()) }; + Some(boolean.into()).map(|value: bool| value.to_string()) + } else { + None + } + }) +} + +fn cf_string(value: &OwnedCf) -> Option { + if unsafe { CFGetTypeID(value.0) } != CFString::type_id() { + return None; + } + // SAFETY: the type id was checked and wrapping under get retains it. + let value = unsafe { CFString::wrap_under_get_rule(value.0.cast()) }; + Some(value.to_string()) +} + +fn attribute_bool(element: AXUIElementRef, attribute_name: &str) -> Option { + let value = copy_attribute(element, attribute_name)?; + if unsafe { CFGetTypeID(value.0) } != CFBoolean::type_id() { + return None; + } + // SAFETY: the type id was checked and wrapping under get retains it. + let boolean = unsafe { CFBoolean::wrap_under_get_rule(value.0.cast()) }; + Some(boolean.into()) +} + +fn element_frame(element: AXUIElementRef) -> Frame { + if let Some(value) = copy_attribute(element, "AXFrame") + && let Some(rect) = ax_rect(&value) + { + return Frame { + x: rect.origin.x, + y: rect.origin.y, + w: rect.size.width, + h: rect.size.height, + }; + } + let point = copy_attribute(element, "AXPosition").and_then(|value| ax_point(&value)); + let size = copy_attribute(element, "AXSize").and_then(|value| ax_size(&value)); + match (point, size) { + (Some(point), Some(size)) => Frame { + x: point.x, + y: point.y, + w: size.width, + h: size.height, + }, + _ => Frame::default(), + } +} + +fn ax_point(value: &OwnedCf) -> Option { + if unsafe { CFGetTypeID(value.0) } != unsafe { AXValueGetTypeID() } + || unsafe { AXValueGetType(value.0) } != AX_VALUE_CGPOINT + { + return None; + } + let mut point = CGPoint::new(0.0, 0.0); + unsafe { + AXValueGetValue( + value.0, + AX_VALUE_CGPOINT, + (&mut point as *mut CGPoint).cast(), + ) + } + .then_some(point) +} + +fn ax_size(value: &OwnedCf) -> Option { + if unsafe { CFGetTypeID(value.0) } != unsafe { AXValueGetTypeID() } + || unsafe { AXValueGetType(value.0) } != AX_VALUE_CGSIZE + { + return None; + } + let mut size = CGSize::new(0.0, 0.0); + unsafe { AXValueGetValue(value.0, AX_VALUE_CGSIZE, (&mut size as *mut CGSize).cast()) } + .then_some(size) +} + +fn ax_rect(value: &OwnedCf) -> Option { + if unsafe { CFGetTypeID(value.0) } != unsafe { AXValueGetTypeID() } + || unsafe { AXValueGetType(value.0) } != AX_VALUE_CGRECT + { + return None; + } + let mut rect = CGRect::new(&CGPoint::new(0.0, 0.0), &CGSize::new(0.0, 0.0)); + unsafe { AXValueGetValue(value.0, AX_VALUE_CGRECT, (&mut rect as *mut CGRect).cast()) } + .then_some(rect) +} + +fn action_names(element: AXUIElementRef) -> Vec { + let mut array: CFArrayRef = ptr::null(); + let code = unsafe { AXUIElementCopyActionNames(element, &mut array) }; + if code != AX_SUCCESS || array.is_null() { + return Vec::new(); + } + // SAFETY: CopyActionNames returns a create-rule CFArray. + let Some(_owner) = (unsafe { OwnedCf::from_create(array as CFTypeRef) }) else { + return Vec::new(); + }; + let count = unsafe { CFArrayGetCount(array) }.max(0) as usize; + let mut actions = Vec::with_capacity(count); + for index in 0..count { + let value = unsafe { CFArrayGetValueAtIndex(array, index as isize) } as CFTypeRef; + if value.is_null() || unsafe { CFGetTypeID(value) } != CFString::type_id() { + continue; + } + // SAFETY: type checked; array owns the borrowed string for this scope. + let value = unsafe { CFString::wrap_under_get_rule(value.cast()) }; + actions.push(canonical_role(&value.to_string())); + } + actions.sort(); + actions.dedup(); + actions +} + +fn element_text(element: AXUIElementRef) -> String { + let mut values = Vec::new(); + for attribute in [ + "AXSelectedText", + "AXValue", + "AXTitle", + "AXDescription", + "AXHelp", + ] { + if let Some(value) = attribute_text(element, attribute) + && !value.is_empty() + && !values.contains(&value) + { + values.push(value); + } + } + values.join("\n") +} + +fn ax_result(operation: &'static str, code: AXError) -> Result<(), AxFailure> { + if code == AX_SUCCESS { + Ok(()) + } else { + Err(AxFailure { operation, code }) + } +} + +fn ax_error_name(code: AXError) -> &'static str { + match code { + 0 => "success", + -25201 => "failure", + -25202 => "illegal argument", + -25203 => "invalid UI element", + -25204 => "invalid observer", + -25205 => "cannot complete", + -25206 => "attribute unsupported", + -25207 => "action unsupported", + -25208 => "notification unsupported", + -25209 => "not implemented", + -25210 => "notification already registered", + -25211 => "notification not registered", + -25212 => "API disabled", + -25213 => "no value", + -25214 => "parameterized attribute unsupported", + -25215 => "not enough precision", + _ => "unknown", + } +} diff --git a/crates/computer-use-mcp/src/backend/macos/capture.rs b/crates/computer-use-mcp/src/backend/macos/capture.rs new file mode 100644 index 00000000..dce5724e --- /dev/null +++ b/crates/computer-use-mcp/src/backend/macos/capture.rs @@ -0,0 +1,169 @@ +use super::super::{BackendError, BackendErrorCode, RootInfo}; + +pub(super) fn capture_window(root: &RootInfo) -> Result, BackendError> { + match capture_window_screen_capture_kit(root) { + Ok(png) => Ok(png), + Err(error) => { + log::warn!( + "computer-use-mcp: ScreenCaptureKit capture failed for window {}: {error}; falling back to screencapture", + root.window_id + ); + capture_window_cli(root) + } + } +} + +fn capture_window_screen_capture_kit(root: &RootInfo) -> Result, String> { + use std::ptr::NonNull; + use std::sync::mpsc; + use std::time::Duration; + + use block2::RcBlock; + use objc2::AnyThread; + use objc2::runtime::AnyClass; + use objc2_core_foundation::{CFMutableData, CFRetained, CFString}; + use objc2_core_graphics::CGImage; + use objc2_foundation::NSError; + use objc2_image_io::CGImageDestination; + use objc2_screen_capture_kit::{ + SCContentFilter, SCScreenshotManager, SCShareableContent, SCStreamConfiguration, + }; + + const CAPTURE_TIMEOUT: Duration = Duration::from_secs(5); + + if AnyClass::get(c"SCScreenshotManager").is_none() { + return Err("SCScreenshotManager is unavailable (requires macOS 14 or newer)".into()); + } + + let window_id = root.window_id; + let (sender, receiver) = mpsc::sync_channel::, String>>(1); + let shareable_content_handler = RcBlock::new( + move |content: *mut SCShareableContent, error: *mut NSError| { + if let Some(error) = unsafe { error.as_ref() } { + let _ = sender.send(Err(format!("failed to get shareable content: {error}"))); + return; + } + let Some(content) = (unsafe { content.as_ref() }) else { + let _ = sender.send(Err("ScreenCaptureKit returned no shareable content".into())); + return; + }; + + let windows = unsafe { content.windows() }; + let Some(window) = windows + .to_vec() + .into_iter() + .find(|window| unsafe { window.windowID() } == window_id) + else { + let _ = sender.send(Err(format!( + "window {window_id} was not present in ScreenCaptureKit shareable content" + ))); + return; + }; + + let filter = unsafe { + SCContentFilter::initWithDesktopIndependentWindow(SCContentFilter::alloc(), &window) + }; + let frame = unsafe { window.frame() }; + let backing_scale = f64::from(unsafe { filter.pointPixelScale() }); + let width = (frame.size.width * backing_scale).round(); + let height = (frame.size.height * backing_scale).round(); + if !width.is_finite() || !height.is_finite() || width < 1.0 || height < 1.0 { + let _ = sender.send(Err(format!( + "invalid ScreenCaptureKit dimensions {width}x{height} for window {window_id}" + ))); + return; + } + + let configuration = unsafe { SCStreamConfiguration::new() }; + unsafe { + configuration.setWidth(width as usize); + configuration.setHeight(height as usize); + configuration.setShowsCursor(false); + } + + let image_sender = sender.clone(); + let image_handler = RcBlock::new(move |image: *mut CGImage, error: *mut NSError| { + let result = if let Some(error) = unsafe { error.as_ref() } { + Err(format!("failed to capture image: {error}")) + } else if let Some(image) = NonNull::new(image) { + Ok(unsafe { CFRetained::retain(image) }) + } else { + Err("ScreenCaptureKit returned no image".into()) + }; + let _ = image_sender.send(result); + }); + unsafe { + SCScreenshotManager::captureImageWithFilter_configuration_completionHandler( + &filter, + &configuration, + Some(&image_handler), + ); + } + }, + ); + + unsafe { + SCShareableContent::getShareableContentWithCompletionHandler(&shareable_content_handler); + } + let image = receiver + .recv_timeout(CAPTURE_TIMEOUT) + .map_err(|error| match error { + mpsc::RecvTimeoutError::Timeout => { + "ScreenCaptureKit capture timed out after 5 seconds".to_owned() + } + mpsc::RecvTimeoutError::Disconnected => { + "ScreenCaptureKit capture callback disconnected".to_owned() + } + })??; + let data = CFMutableData::new(None, 0) + .ok_or_else(|| "failed to allocate PNG destination data".to_owned())?; + let png_type = CFString::from_static_str("public.png"); + let destination = unsafe { CGImageDestination::with_data(&data, &png_type, 1, None) } + .ok_or_else(|| "failed to create PNG image destination".to_owned())?; + unsafe { + destination.add_image(&image, None); + if !destination.finalize() { + return Err("failed to finalize ScreenCaptureKit PNG".into()); + } + } + Ok(data.to_vec()) +} + +fn capture_window_cli(root: &RootInfo) -> Result, BackendError> { + let path = std::env::temp_dir().join(format!( + "tcode-computer-use-{}-{}.png", + root.window_id, + uuid::Uuid::new_v4() + )); + let result = tcode_services::process::command("screencapture") + .arg("-x") + .arg("-l") + .arg(root.window_id.to_string()) + .arg("-t") + .arg("png") + .arg(&path) + .status() + .map_err(|error| { + BackendError::new( + BackendErrorCode::CaptureFailed, + format!("failed to spawn screencapture: {error}"), + ) + }) + .and_then(|status| { + if status.success() { + std::fs::read(&path).map_err(|error| { + BackendError::new( + BackendErrorCode::CaptureFailed, + format!("failed to read captured PNG: {error}"), + ) + }) + } else { + Err(BackendError::new( + BackendErrorCode::CaptureFailed, + format!("screencapture exited with status {status}"), + )) + } + }); + let _ = std::fs::remove_file(path); + result +} diff --git a/crates/computer-use-mcp/src/backend/macos/input.rs b/crates/computer-use-mcp/src/backend/macos/input.rs new file mode 100644 index 00000000..bcf73cda --- /dev/null +++ b/crates/computer-use-mcp/src/backend/macos/input.rs @@ -0,0 +1,232 @@ +use std::time::Duration; + +use core_graphics::event::{ + CGEvent, CGEventFlags, CGEventTapLocation, CGEventType, CGMouseButton, EventField, + ScrollEventUnit, +}; +use core_graphics::event_source::{CGEventSource, CGEventSourceStateID}; +use core_graphics::geometry::CGPoint; + +use super::super::{BackendError, BackendErrorCode, KeyModifiers, MouseButton, parse_key_chord}; + +pub(super) fn click( + x: f64, + y: f64, + button: MouseButton, + click_count: u32, +) -> Result<(), BackendError> { + if !x.is_finite() || !y.is_finite() { + return Err(invalid("click coordinates must be finite")); + } + if !(1..=3).contains(&click_count) { + return Err(invalid("click_count must be between 1 and 3")); + } + let point = CGPoint::new(x, y); + let (down, up, cg_button) = mouse_types(button); + for click_index in 0..click_count { + let down_event = mouse_event(down, point, cg_button)?; + down_event.set_integer_value_field(EventField::MOUSE_EVENT_CLICK_STATE, click_count.into()); + down_event.post(CGEventTapLocation::HID); + let up_event = mouse_event(up, point, cg_button)?; + up_event.set_integer_value_field(EventField::MOUSE_EVENT_CLICK_STATE, click_count.into()); + up_event.post(CGEventTapLocation::HID); + if click_index + 1 < click_count { + std::thread::sleep(Duration::from_millis(45)); + } + } + Ok(()) +} + +pub(super) fn move_mouse(x: f64, y: f64) -> Result<(), BackendError> { + if !x.is_finite() || !y.is_finite() { + return Err(invalid("mouse coordinates must be finite")); + } + mouse_event( + CGEventType::MouseMoved, + CGPoint::new(x, y), + CGMouseButton::Left, + )? + .post(CGEventTapLocation::HID); + Ok(()) +} + +pub(super) fn scroll(x: f64, y: f64) -> Result<(), BackendError> { + if !x.is_finite() || !y.is_finite() { + return Err(invalid("scroll deltas must be finite")); + } + let event = CGEvent::new_scroll_event( + event_source()?, + ScrollEventUnit::PIXEL, + 2, + y.round() as i32, + x.round() as i32, + 0, + ) + .map_err(|()| operation("CoreGraphics could not create a scroll event"))?; + event.post(CGEventTapLocation::HID); + Ok(()) +} + +pub(super) fn drag(path: &[[f64; 2]], button: MouseButton) -> Result<(), BackendError> { + if path.len() < 2 { + return Err(invalid("drag requires at least two path points")); + } + if path + .iter() + .flatten() + .any(|coordinate| !coordinate.is_finite()) + { + return Err(invalid("drag path coordinates must be finite")); + } + let (_, up, cg_button) = mouse_types(button); + let down = match button { + MouseButton::Left => CGEventType::LeftMouseDown, + MouseButton::Right => CGEventType::RightMouseDown, + MouseButton::Middle => CGEventType::OtherMouseDown, + }; + let dragged = match button { + MouseButton::Left => CGEventType::LeftMouseDragged, + MouseButton::Right => CGEventType::RightMouseDragged, + MouseButton::Middle => CGEventType::OtherMouseDragged, + }; + let first = CGPoint::new(path[0][0], path[0][1]); + mouse_event(down, first, cg_button)?.post(CGEventTapLocation::HID); + for point in &path[1..path.len() - 1] { + mouse_event(dragged, CGPoint::new(point[0], point[1]), cg_button)? + .post(CGEventTapLocation::HID); + std::thread::sleep(Duration::from_millis(12)); + } + let last = path[path.len() - 1]; + let last = CGPoint::new(last[0], last[1]); + mouse_event(dragged, last, cg_button)?.post(CGEventTapLocation::HID); + mouse_event(up, last, cg_button)?.post(CGEventTapLocation::HID); + Ok(()) +} + +pub(super) fn keypress(keys: &[String]) -> Result<(), BackendError> { + let chord = parse_key_chord(keys).map_err(invalid)?; + let flags = modifier_flags(chord.modifiers); + let down = CGEvent::new_keyboard_event(event_source()?, chord.keycode, true) + .map_err(|()| operation("CoreGraphics could not create a key-down event"))?; + down.set_flags(flags); + down.post(CGEventTapLocation::HID); + let up = CGEvent::new_keyboard_event(event_source()?, chord.keycode, false) + .map_err(|()| operation("CoreGraphics could not create a key-up event"))?; + up.set_flags(flags); + up.post(CGEventTapLocation::HID); + Ok(()) +} + +pub(super) fn type_text(text: &str) -> Result<(), BackendError> { + let chunks = unicode_chunks(text, 20); + for chunk in chunks { + let down = CGEvent::new_keyboard_event(event_source()?, 0, true) + .map_err(|()| operation("CoreGraphics could not create a Unicode key-down event"))?; + down.set_flags(CGEventFlags::CGEventFlagNull); + down.set_string_from_utf16_unchecked(&chunk); + down.post(CGEventTapLocation::HID); + let up = CGEvent::new_keyboard_event(event_source()?, 0, false) + .map_err(|()| operation("CoreGraphics could not create a Unicode key-up event"))?; + up.set_flags(CGEventFlags::CGEventFlagNull); + up.set_string_from_utf16_unchecked(&chunk); + up.post(CGEventTapLocation::HID); + std::thread::sleep(Duration::from_millis(5)); + } + Ok(()) +} + +fn unicode_chunks(text: &str, maximum_units: usize) -> Vec> { + let mut chunks = Vec::new(); + let mut current = Vec::new(); + for character in text.chars() { + let mut encoded = [0_u16; 2]; + let units = character.encode_utf16(&mut encoded); + if !current.is_empty() && current.len() + units.len() > maximum_units { + chunks.push(std::mem::take(&mut current)); + } + current.extend_from_slice(units); + } + if !current.is_empty() { + chunks.push(current); + } + chunks +} + +fn event_source() -> Result { + CGEventSource::new(CGEventSourceStateID::HIDSystemState) + .map_err(|()| operation("CoreGraphics could not create an event source")) +} + +fn mouse_event( + event_type: CGEventType, + point: CGPoint, + button: CGMouseButton, +) -> Result { + CGEvent::new_mouse_event(event_source()?, event_type, point, button) + .map_err(|()| operation("CoreGraphics could not create a mouse event")) +} + +fn mouse_types(button: MouseButton) -> (CGEventType, CGEventType, CGMouseButton) { + match button { + MouseButton::Left => ( + CGEventType::LeftMouseDown, + CGEventType::LeftMouseUp, + CGMouseButton::Left, + ), + MouseButton::Right => ( + CGEventType::RightMouseDown, + CGEventType::RightMouseUp, + CGMouseButton::Right, + ), + MouseButton::Middle => ( + CGEventType::OtherMouseDown, + CGEventType::OtherMouseUp, + CGMouseButton::Center, + ), + } +} + +fn modifier_flags(modifiers: KeyModifiers) -> CGEventFlags { + let mut flags = CGEventFlags::CGEventFlagNull; + if modifiers.command { + flags |= CGEventFlags::CGEventFlagCommand; + } + if modifiers.control { + flags |= CGEventFlags::CGEventFlagControl; + } + if modifiers.option { + flags |= CGEventFlags::CGEventFlagAlternate; + } + if modifiers.shift { + flags |= CGEventFlags::CGEventFlagShift; + } + if modifiers.function { + flags |= CGEventFlags::CGEventFlagSecondaryFn; + } + flags +} + +fn invalid(message: impl Into) -> BackendError { + BackendError::new(BackendErrorCode::InvalidAction, message) +} + +fn operation(message: impl Into) -> BackendError { + BackendError::new(BackendErrorCode::OperationFailed, message) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unicode_chunks_do_not_split_surrogate_pairs() { + let chunks = unicode_chunks("1234567890123456789🦀x", 20); + assert_eq!(chunks.len(), 2); + assert!(chunks.iter().all(|chunk| chunk.len() <= 20)); + let decoded: String = chunks + .into_iter() + .flat_map(|chunk| char::decode_utf16(chunk).map(Result::unwrap)) + .collect(); + assert_eq!(decoded, "1234567890123456789🦀x"); + } +} diff --git a/crates/computer-use-mcp/src/backend/macos/mod.rs b/crates/computer-use-mcp/src/backend/macos/mod.rs new file mode 100644 index 00000000..8a961a02 --- /dev/null +++ b/crates/computer-use-mcp/src/backend/macos/mod.rs @@ -0,0 +1,362 @@ +mod ax; +mod capture; +mod input; + +use std::collections::HashMap; +use std::ffi::c_void; + +use core_foundation::array::{CFArrayGetCount, CFArrayGetValueAtIndex}; +use core_foundation::base::{CFGetTypeID, CFTypeRef, TCFType}; +use core_foundation::dictionary::{CFDictionaryGetTypeID, CFDictionaryGetValue, CFDictionaryRef}; +use core_foundation::number::CFNumber; +use core_foundation::string::{CFString, CFStringRef}; +use core_graphics::window::{ + kCGNullWindowID, kCGWindowBounds, kCGWindowLayer, kCGWindowListExcludeDesktopElements, + kCGWindowListOptionOnScreenOnly, kCGWindowName, kCGWindowNumber, kCGWindowOwnerName, + kCGWindowOwnerPID, +}; + +use super::{ + ActionKind, ActionRequest, ActionResult, Backend, BackendError, BackendErrorCode, + CapturePolicy, ObserveRequest, RootFilters, RootInfo, RootObservation, +}; +use crate::outline::{UiNode, interactive_count}; + +pub(super) struct MacosBackend; + +impl Backend for MacosBackend { + fn list_roots(&self, filters: &RootFilters) -> Result, BackendError> { + let options = kCGWindowListOptionOnScreenOnly | kCGWindowListExcludeDesktopElements; + let array = + core_graphics::window::copy_window_info(options, kCGNullWindowID).ok_or_else(|| { + BackendError::new( + BackendErrorCode::ObservationFailed, + "CGWindowListCopyWindowInfo returned no window list", + ) + })?; + let array_ref = array.as_concrete_TypeRef(); + let count = unsafe { CFArrayGetCount(array_ref) }.max(0) as usize; + let mut roots = Vec::new(); + let mut identifiers = HashMap::::new(); + for index in 0..count { + let dictionary = + unsafe { CFArrayGetValueAtIndex(array_ref, index as isize) } as CFDictionaryRef; + if dictionary.is_null() + || unsafe { CFGetTypeID(dictionary.cast()) } != unsafe { CFDictionaryGetTypeID() } + { + continue; + } + let layer = dictionary_i64(dictionary, unsafe { kCGWindowLayer }).unwrap_or(-1); + if layer != 0 { + continue; + } + let pid = dictionary_i64(dictionary, unsafe { kCGWindowOwnerPID }) + .and_then(|pid| u32::try_from(pid).ok()) + .unwrap_or_default(); + let window_id = dictionary_i64(dictionary, unsafe { kCGWindowNumber }) + .and_then(|id| u32::try_from(id).ok()) + .unwrap_or_default(); + if pid == 0 || window_id == 0 { + continue; + } + let app_name = + dictionary_string(dictionary, unsafe { kCGWindowOwnerName }).unwrap_or_default(); + let title = dictionary_string(dictionary, unsafe { kCGWindowName }).unwrap_or_default(); + let frame = dictionary_value(dictionary, unsafe { kCGWindowBounds }) + .and_then(|value| window_bounds(value.cast())) + .unwrap_or_default(); + if !frame.has_area() { + continue; + } + let bundle_id = identifiers + .entry(pid) + .or_insert_with(|| ax::application_identifier(pid)) + .clone(); + let mut root = RootInfo { + ref_id: String::new(), + app_name, + bundle_id, + pid, + title, + kind: super::RootKind::Window, + window_id, + frame, + }; + root.kind = ax::root_kind(&root); + if matches_filters(&root, filters) { + roots.push(root); + } + } + // CGWindowListCopyWindowInfo is documented front-to-back. Preserve that + // ordering so the first result is the frontmost eligible root. + Ok(roots) + } + + fn observe( + &self, + root: &RootInfo, + request: ObserveRequest, + ) -> Result { + let tree = if request.semantic { + ax::observe_tree(root)? + } else { + UiNode { + role: root.kind.to_string(), + title: root.title.clone(), + frame: root.frame, + enabled: true, + ..UiNode::default() + } + }; + let should_capture = match request.capture { + CapturePolicy::Never => false, + CapturePolicy::Always => true, + CapturePolicy::IfSparse => interactive_count(&tree) <= 3, + }; + let screenshot_png = should_capture + .then(|| capture::capture_window(root)) + .transpose()?; + Ok(RootObservation { + root: root.clone(), + tree, + screenshot_png, + }) + } + + fn perform_action( + &self, + root: &RootInfo, + request: &ActionRequest, + ) -> Result { + match request.kind { + ActionKind::Press => { + let target = target(root, request)?; + Ok(match target.press() { + Ok(()) => ActionResult::worked("AXPress completed"), + Err(error) => ActionResult::didnt(error.to_string()), + }) + } + ActionKind::Click => { + if request.target_path.is_some() + && request.target_actions.iter().any(|a| a == "press") + { + let target = target(root, request)?; + if target.press().is_ok() { + return Ok(ActionResult::worked("AXPress completed for click target")); + } + let (x, y) = target.frame().center(); + input::click(x, y, request.button, request.click_count)?; + return Ok(ActionResult::unknown( + "AXPress was rejected; physical click events were posted", + )); + } + let (x, y) = action_point(root, request)?; + input::click(x, y, request.button, request.click_count)?; + Ok(ActionResult::unknown("physical click events were posted")) + } + ActionKind::SetText => { + let text = request.text.as_deref().ok_or_else(|| { + BackendError::new(BackendErrorCode::InvalidAction, "set_text requires text") + })?; + let target = target(root, request)?; + match target.set_text(text) { + Ok(()) => Ok(ActionResult::worked("AXValue was set")), + Err(ax_error) => { + if target.focus().is_err() { + let frame = target.frame(); + if !frame.has_area() { + return Ok(ActionResult::didnt(format!( + "{ax_error}; the target also rejected focus and has no clickable frame" + ))); + } + let (x, y) = frame.center(); + input::click(x, y, super::MouseButton::Left, 1)?; + } + input::keypress(&["cmd+a".into()])?; + input::type_text(text)?; + Ok(ActionResult::unknown(format!( + "{ax_error}; keyboard replacement events were posted instead" + ))) + } + } + } + ActionKind::TypeText => { + let text = request.text.as_deref().ok_or_else(|| { + BackendError::new(BackendErrorCode::InvalidAction, "type_text requires text") + })?; + if request.target_path.is_some() { + let target = target(root, request)?; + if target.focus().is_err() { + let frame = target.frame(); + if !frame.has_area() { + return Ok(ActionResult::didnt( + "target rejected focus and has no clickable frame", + )); + } + let (x, y) = frame.center(); + input::click(x, y, super::MouseButton::Left, 1)?; + } + } + input::type_text(text)?; + Ok(ActionResult::unknown("Unicode keyboard events were posted")) + } + ActionKind::Keypress => { + if request.target_path.is_some() { + let target = target(root, request)?; + if target.focus().is_err() { + let frame = target.frame(); + if !frame.has_area() { + return Ok(ActionResult::didnt( + "keypress target rejected focus and has no clickable frame", + )); + } + let (x, y) = frame.center(); + input::click(x, y, super::MouseButton::Left, 1)?; + } + } + let keys = request.keys.as_deref().ok_or_else(|| { + BackendError::new(BackendErrorCode::InvalidAction, "keypress requires keys") + })?; + input::keypress(keys)?; + Ok(ActionResult::unknown("keyboard events were posted")) + } + ActionKind::Scroll => { + if request.target_path.is_some() || (request.x.is_some() && request.y.is_some()) { + let (x, y) = action_point(root, request)?; + input::move_mouse(x, y)?; + } + input::scroll( + request.scroll_x.unwrap_or(0.0), + request.scroll_y.unwrap_or(0.0), + )?; + Ok(ActionResult::unknown("scroll-wheel events were posted")) + } + ActionKind::Drag => { + let path = request.path.as_deref().ok_or_else(|| { + BackendError::new(BackendErrorCode::InvalidAction, "drag requires a path") + })?; + input::drag(path, request.button)?; + Ok(ActionResult::unknown("drag events were posted")) + } + ActionKind::MoveMouse => { + let (x, y) = action_point(root, request)?; + input::move_mouse(x, y)?; + Ok(ActionResult::unknown("mouse-move event was posted")) + } + } + } + + fn read_element_text( + &self, + root: &RootInfo, + target_path: &[usize], + ) -> Result { + ax::read_target_text(root, target_path) + } +} + +fn target(root: &RootInfo, request: &ActionRequest) -> Result { + let path = request.target_path.as_deref().ok_or_else(|| { + BackendError::new( + BackendErrorCode::InvalidAction, + "this action requires an element ref", + ) + })?; + ax::locate_target( + root, + path, + request.target_role.as_deref(), + request.target_title.as_deref(), + ) +} + +fn action_point(root: &RootInfo, request: &ActionRequest) -> Result<(f64, f64), BackendError> { + if let (Some(x), Some(y)) = (request.x, request.y) { + return Ok((x, y)); + } + if request.target_path.is_some() { + let live_frame = target(root, request)?.frame(); + if live_frame.has_area() { + return Ok(live_frame.center()); + } + } + if let Some(frame) = request.target_frame.filter(|frame| frame.has_area()) { + return Ok(frame.center()); + } + Err(BackendError::new( + BackendErrorCode::InvalidAction, + "action requires x/y coordinates or an element with a non-empty frame", + )) +} + +fn matches_filters(root: &RootInfo, filters: &RootFilters) -> bool { + if filters.pid.is_some_and(|pid| root.pid != pid) + || filters.kind.is_some_and(|kind| root.kind != kind) + { + return false; + } + if filters + .app + .as_deref() + .is_some_and(|app| !contains_case_insensitive(&root.app_name, app)) + || filters + .bundle_id + .as_deref() + .is_some_and(|bundle| !contains_case_insensitive(&root.bundle_id, bundle)) + { + return false; + } + filters.text.as_deref().is_none_or(|text| { + contains_case_insensitive(&root.app_name, text) + || contains_case_insensitive(&root.title, text) + || contains_case_insensitive(&root.bundle_id, text) + }) +} + +fn contains_case_insensitive(haystack: &str, needle: &str) -> bool { + haystack.to_lowercase().contains(&needle.to_lowercase()) +} + +fn dictionary_value(dictionary: CFDictionaryRef, key: CFStringRef) -> Option<*const c_void> { + let value = unsafe { CFDictionaryGetValue(dictionary, key.cast()) }; + (!value.is_null()).then_some(value) +} + +fn dictionary_i64(dictionary: CFDictionaryRef, key: CFStringRef) -> Option { + let value = dictionary_value(dictionary, key)? as CFTypeRef; + if unsafe { CFGetTypeID(value) } != CFNumber::type_id() { + return None; + } + // SAFETY: type checked; dictionary owns the borrowed value for this scope. + unsafe { CFNumber::wrap_under_get_rule(value.cast()) }.to_i64() +} + +fn dictionary_string(dictionary: CFDictionaryRef, key: CFStringRef) -> Option { + let value = dictionary_value(dictionary, key)? as CFTypeRef; + if unsafe { CFGetTypeID(value) } != CFString::type_id() { + return None; + } + // SAFETY: type checked; dictionary owns the borrowed value for this scope. + Some(unsafe { CFString::wrap_under_get_rule(value.cast()) }.to_string()) +} + +fn window_bounds(dictionary: CFDictionaryRef) -> Option { + if unsafe { CFGetTypeID(dictionary.cast()) } != unsafe { CFDictionaryGetTypeID() } { + return None; + } + let number = |key: &str| { + let key = CFString::new(key); + let value = dictionary_value(dictionary, key.as_concrete_TypeRef())? as CFTypeRef; + if unsafe { CFGetTypeID(value) } != CFNumber::type_id() { + return None; + } + // SAFETY: type checked; dictionary owns the borrowed value for this scope. + unsafe { CFNumber::wrap_under_get_rule(value.cast()) }.to_f64() + }; + Some(crate::outline::Frame { + x: number("X")?, + y: number("Y")?, + w: number("Width")?, + h: number("Height")?, + }) +} diff --git a/crates/computer-use-mcp/src/backend/stub.rs b/crates/computer-use-mcp/src/backend/stub.rs new file mode 100644 index 00000000..bebb4b59 --- /dev/null +++ b/crates/computer-use-mcp/src/backend/stub.rs @@ -0,0 +1,36 @@ +use super::{ + ActionRequest, ActionResult, Backend, BackendError, ObserveRequest, RootFilters, RootInfo, + RootObservation, +}; + +pub(super) struct StubBackend; + +impl Backend for StubBackend { + fn list_roots(&self, _filters: &RootFilters) -> Result, BackendError> { + Err(BackendError::unsupported()) + } + + fn observe( + &self, + _root: &RootInfo, + _request: ObserveRequest, + ) -> Result { + Err(BackendError::unsupported()) + } + + fn perform_action( + &self, + _root: &RootInfo, + _request: &ActionRequest, + ) -> Result { + Err(BackendError::unsupported()) + } + + fn read_element_text( + &self, + _root: &RootInfo, + _target_path: &[usize], + ) -> Result { + Err(BackendError::unsupported()) + } +} diff --git a/crates/computer-use-mcp/src/bin/cu-smoke.rs b/crates/computer-use-mcp/src/bin/cu-smoke.rs new file mode 100644 index 00000000..155d43f7 --- /dev/null +++ b/crates/computer-use-mcp/src/bin/cu-smoke.rs @@ -0,0 +1,93 @@ +use std::path::PathBuf; + +fn print_json(value: &impl serde::Serialize) { + println!( + "{}", + serde_json::to_string(value).expect("debug tool result must serialize") + ); +} + +fn take_argument(arguments: &mut impl Iterator, flag: &str) -> String { + arguments + .next() + .unwrap_or_else(|| panic!("{flag} requires an argument")) +} + +fn main() { + let mut arguments = std::env::args().skip(1).peekable(); + if arguments + .peek() + .is_some_and(|argument| argument == "--permissions") + { + let status = computer_use_mcp::permissions::check(); + print_json(&status); + return; + } + + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .expect("failed to create smoke-test runtime"); + + if arguments + .peek() + .is_some_and(|argument| argument.starts_with("--")) + { + let result = runtime.block_on(async { + let mut last_root: Option = None; + while let Some(flag) = arguments.next() { + match flag.as_str() { + "--find-roots" => { + let filter = arguments + .next_if(|argument| !argument.starts_with("--")) + .unwrap_or_else(|| "{}".into()); + let result = computer_use_mcp::tools::debug_find_roots(&filter) + .await + .map_err(|error| error.to_string())?; + print_json(&result); + } + "--observe" => { + let root = take_argument(&mut arguments, "--observe"); + last_root = (!root.is_empty()).then_some(root.clone()); + print_json(&computer_use_mcp::tools::debug_observe(&root).await); + } + "--search" => { + let state_id = take_argument(&mut arguments, "--search"); + let text = take_argument(&mut arguments, "--search"); + print_json(&computer_use_mcp::tools::debug_search(&state_id, &text).await); + } + "--act" => { + let params = take_argument(&mut arguments, "--act"); + let result = computer_use_mcp::tools::debug_act(¶ms) + .await + .map_err(|error| error.to_string())?; + print_json(&result); + } + "--screenshot" => { + let output = PathBuf::from(take_argument(&mut arguments, "--screenshot")); + let screenshot = + computer_use_mcp::tools::debug_screenshot(last_root.as_deref()).await; + if let Some(png) = screenshot.png { + std::fs::write(&output, png).map_err(|error| error.to_string())?; + } + print_json(&serde_json::json!({ + "output": output, + "tool_result": screenshot.result, + })); + } + unknown => return Err(format!("unknown debug subcommand: {unknown}")), + } + } + Ok::<(), String>(()) + }); + if let Err(error) = result { + print_json(&serde_json::json!({ "error": error })); + std::process::exit(2); + } + return; + } + + let run = runtime.block_on(computer_use_mcp::tools::run_smoke()); + print_json(&run.verdict); + std::process::exit(run.exit_code); +} diff --git a/crates/computer-use-mcp/src/config.rs b/crates/computer-use-mcp/src/config.rs new file mode 100644 index 00000000..9ae418ab --- /dev/null +++ b/crates/computer-use-mcp/src/config.rs @@ -0,0 +1,44 @@ +//! Live feature configuration, pushed by the app whenever settings load or +//! change. The MCP server outlives any single settings snapshot, so tools read +//! the current value at call time instead of capturing one at startup. + +use std::sync::RwLock; + +/// When observations include a screenshot alongside the outline. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum ImageMode { + /// Screenshot only when the accessibility outline looks too sparse to act on. + #[default] + Auto, + Always, + Never, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ComputerUseConfig { + /// When false the tools are observe-only: `act_ui` rejects every action. + pub allow_input: bool, + pub image_mode: ImageMode, +} + +impl Default for ComputerUseConfig { + fn default() -> Self { + Self { + allow_input: true, + image_mode: ImageMode::Auto, + } + } +} + +static CONFIG: RwLock = RwLock::new(ComputerUseConfig { + allow_input: true, + image_mode: ImageMode::Auto, +}); + +pub fn set(config: ComputerUseConfig) { + *CONFIG.write().unwrap() = config; +} + +pub fn get() -> ComputerUseConfig { + *CONFIG.read().unwrap() +} diff --git a/crates/computer-use-mcp/src/lib.rs b/crates/computer-use-mcp/src/lib.rs new file mode 100644 index 00000000..432c4529 --- /dev/null +++ b/crates/computer-use-mcp/src/lib.rs @@ -0,0 +1,64 @@ +//! In-process `tcode_computer_use` MCP server: pi-computer-use-style desktop +//! automation for every provider (accessibility-tree observation, state-scoped +//! refs, transactional actions). See `docs/computer-use.md` for the design. +//! +//! Served over streamable HTTP on `127.0.0.1:` with a bearer +//! token, mirroring `preview-mcp` / `orchestrate-mcp`. The macOS backend talks +//! to the AX C API, CGEvent, and `screencapture`; other platforms serve a stub +//! that reports the platform as unsupported. + +pub mod backend; +pub mod config; +pub mod outline; +pub mod permissions; +pub mod state; +pub mod tools; + +use std::sync::{Arc, RwLock}; + +/// A running computer-use MCP server and the bearer token required to access it. +pub struct ComputerUseMcpServer { + /// Streamable-HTTP endpoint, e.g. `http://127.0.0.1:53211/mcp`. + pub url: String, + /// Bearer token presented by every registered provider session. + pub token: String, +} + +/// Bind a random loopback port and start the authenticated streamable-HTTP MCP +/// server on a dedicated tokio runtime thread. +pub fn start() -> std::io::Result { + let listener = std::net::TcpListener::bind(("127.0.0.1", 0))?; + let port = listener.local_addr()?.port(); + let url = format!("http://127.0.0.1:{port}/mcp"); + let token = format!( + "{}{}", + uuid::Uuid::new_v4().simple(), + uuid::Uuid::new_v4().simple() + ); + let mut services = tools::Services::new(); + services.insert(token.clone(), tools::service()); + let services = Arc::new(RwLock::new(services)); + + std::thread::Builder::new() + .name("computer-use-mcp".into()) + .spawn(move || { + let runtime = match tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(err) => { + log::error!("computer-use-mcp: failed to build tokio runtime: {err}"); + return; + } + }; + runtime.block_on(async move { + if let Err(err) = tools::serve(listener, services).await { + log::error!("computer-use-mcp: server exited with error: {err}"); + } + }); + })?; + + log::info!("computer-use-mcp: serving at {url}"); + Ok(ComputerUseMcpServer { url, token }) +} diff --git a/crates/computer-use-mcp/src/outline.rs b/crates/computer-use-mcp/src/outline.rs new file mode 100644 index 00000000..98fe022e --- /dev/null +++ b/crates/computer-use-mcp/src/outline.rs @@ -0,0 +1,748 @@ +//! Platform-neutral accessibility tree, progressive rendering, and diffing. + +use std::collections::{HashMap, HashSet}; + +use serde::{Deserialize, Serialize}; + +pub const MAX_MODEL_BYTES: usize = 48 * 1024; +pub const MAX_MODEL_LINES: usize = 2_000; +pub const PREVIEW_BYTES: usize = 16 * 1024; +pub const PAGE_BYTES: usize = 16 * 1024; +pub const SEARCH_LIMIT: usize = 20; + +const FOLDED_DEPTH: usize = 7; +const FOLDED_LINES: usize = 500; +const EXPANDED_LINES: usize = 1_000; + +#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)] +pub struct Frame { + pub x: f64, + pub y: f64, + pub w: f64, + pub h: f64, +} + +impl Frame { + pub fn center(self) -> (f64, f64) { + (self.x + self.w / 2.0, self.y + self.h / 2.0) + } + + pub fn has_area(self) -> bool { + self.w > 0.0 && self.h > 0.0 + } + + pub fn intersects(self, other: Self) -> bool { + self.has_area() + && other.has_area() + && self.x < other.x + other.w + && self.x + self.w > other.x + && self.y < other.y + other.h + && self.y + self.h > other.y + } +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +pub struct UiNode { + pub ref_id: String, + pub role: String, + pub title: String, + pub value: String, + pub description: String, + pub frame: Frame, + pub actions: Vec, + pub enabled: bool, + pub focused: bool, + pub children: Vec, +} + +impl UiNode { + pub fn is_interactive(&self) -> bool { + is_interactive_role(&self.role) || !self.actions.is_empty() + } + + pub fn text(&self) -> String { + let mut parts = Vec::new(); + for value in [&self.value, &self.title, &self.description] { + if !value.is_empty() && !parts.contains(&value) { + parts.push(value); + } + } + parts + .into_iter() + .map(String::as_str) + .collect::>() + .join("\n") + } + + pub fn find(&self, ref_id: &str) -> Option<&Self> { + if self.ref_id == ref_id { + return Some(self); + } + self.children.iter().find_map(|child| child.find(ref_id)) + } + + pub fn find_mut(&mut self, ref_id: &str) -> Option<&mut Self> { + if self.ref_id == ref_id { + return Some(self); + } + self.children + .iter_mut() + .find_map(|child| child.find_mut(ref_id)) + } + + pub fn at_path(&self, path: &[usize]) -> Option<&Self> { + let mut node = self; + for &index in path { + node = node.children.get(index)?; + } + Some(node) + } +} + +pub fn is_interactive_role(role: &str) -> bool { + matches!( + canonical_role(role).as_str(), + "button" + | "text_field" + | "text_area" + | "search_field" + | "link" + | "menu" + | "menu_item" + | "checkbox" + | "radio_button" + | "combo_box" + | "pop_up_button" + | "slider" + | "incrementor" + | "tab" + ) +} + +pub fn canonical_role(role: &str) -> String { + role.trim() + .strip_prefix("AX") + .unwrap_or(role.trim()) + .chars() + .enumerate() + .fold(String::new(), |mut result, (index, ch)| { + if ch.is_ascii_uppercase() && index != 0 { + result.push('_'); + } + result.push(ch.to_ascii_lowercase()); + result + }) +} + +pub fn interactive_count(root: &UiNode) -> usize { + let mut count = usize::from(root.is_interactive()); + for child in &root.children { + count += interactive_count(child); + } + count +} + +pub fn assign_refs(root: &mut UiNode) { + let mut next = 1_u64; + walk_mut(root, &mut |node, _| { + node.ref_id = format!("@e{next}"); + next += 1; + }); +} + +/// Preserve refs when a successor has the same role/title at the same path, +/// then use unique role/title matches before allocating fresh refs. +pub fn assign_refs_from_previous(previous: &UiNode, successor: &mut UiNode) { + let old = flatten(previous); + let mut by_path = HashMap::new(); + let mut by_signature: HashMap<(String, String), Vec> = HashMap::new(); + let mut next = 1_u64; + for entry in &old { + by_path.insert( + entry.path.clone(), + (entry.signature.clone(), entry.ref_id.clone()), + ); + by_signature + .entry(entry.signature.clone()) + .or_default() + .push(entry.ref_id.clone()); + next = next.max(ref_number(&entry.ref_id).unwrap_or(0) + 1); + } + + let mut used = HashSet::new(); + walk_mut(successor, &mut |node, path| { + let signature = node_signature(node); + let path_match = by_path + .get(path) + .filter(|(old_signature, _)| old_signature == &signature) + .map(|(_, ref_id)| ref_id.clone()); + let unique_match = by_signature.get(&signature).and_then(|refs| { + let mut available = refs.iter().filter(|ref_id| !used.contains(*ref_id)); + let first = available.next()?; + available.next().is_none().then(|| first.clone()) + }); + let ref_id = path_match + .filter(|ref_id| !used.contains(ref_id)) + .or(unique_match) + .unwrap_or_else(|| { + let allocated = format!("@e{next}"); + next += 1; + allocated + }); + used.insert(ref_id.clone()); + node.ref_id = ref_id; + }); +} + +pub fn path_to_ref(root: &UiNode, ref_id: &str) -> Option> { + fn find(node: &UiNode, wanted: &str, path: &mut Vec) -> bool { + if node.ref_id == wanted { + return true; + } + for (index, child) in node.children.iter().enumerate() { + path.push(index); + if find(child, wanted, path) { + return true; + } + path.pop(); + } + false + } + + let mut path = Vec::new(); + find(root, ref_id, &mut path).then_some(path) +} + +pub fn render_folded(root: &UiNode) -> String { + let mut output = Vec::new(); + render_folded_node(root, 0, true, &mut output); + output.truncate(FOLDED_LINES); + if output.len() == FOLDED_LINES { + output.push("… outline capped; use search_ui or expand_ui for the full cached tree".into()); + } + output.join("\n") +} + +fn render_folded_node(node: &UiNode, depth: usize, is_root: bool, output: &mut Vec) { + if output.len() >= FOLDED_LINES { + return; + } + if !is_root && is_collapsible_container(node) && node.children.len() == 1 { + render_folded_node(&node.children[0], depth, false, output); + return; + } + + output.push(render_line(node, depth)); + if depth < FOLDED_DEPTH { + for child in &node.children { + render_folded_node(child, depth + 1, false, output); + } + return; + } + + let before = output.len(); + for child in &node.children { + render_interactive_descendants(child, depth + 1, output); + } + if before == output.len() && !node.children.is_empty() && output.len() < FOLDED_LINES { + output.push(format!( + "{}… {} descendants folded", + " ".repeat(depth + 1), + descendant_count(node) + )); + } +} + +fn render_interactive_descendants(node: &UiNode, depth: usize, output: &mut Vec) { + if output.len() >= FOLDED_LINES { + return; + } + if node.is_interactive() { + output.push(render_line(node, depth)); + } + for child in &node.children { + render_interactive_descendants(child, depth, output); + } +} + +pub fn render_expanded(root: &UiNode, ref_id: &str, depth: usize) -> Result { + let path = path_to_ref(root, ref_id) + .ok_or_else(|| format!("element ref {ref_id} is not owned by this state"))?; + let mut lines = Vec::new(); + let mut node = root; + lines.push(render_line(node, 0)); + for (level, &index) in path.iter().enumerate() { + node = &node.children[index]; + lines.push(render_line(node, level + 1)); + } + render_subtree_children(node, path.len() + 1, depth, &mut lines); + lines.truncate(EXPANDED_LINES); + if lines.len() == EXPANDED_LINES { + lines.push("… expansion capped; narrow the depth or use search_ui".into()); + } + Ok(lines.join("\n")) +} + +fn render_subtree_children(node: &UiNode, indent: usize, depth: usize, lines: &mut Vec) { + if depth == 0 || lines.len() >= EXPANDED_LINES { + return; + } + for child in &node.children { + lines.push(render_line(child, indent)); + render_subtree_children(child, indent + 1, depth - 1, lines); + } +} + +pub fn render_line(node: &UiNode, depth: usize) -> String { + let mut line = format!( + "{}{} {}", + " ".repeat(depth), + node.ref_id, + canonical_role(&node.role) + ); + if !node.title.is_empty() { + line.push_str(&format!(" \"{}\"", display_string(&node.title, 240))); + } + if !node.value.is_empty() && node.value != node.title { + line.push_str(&format!(" value=\"{}\"", display_string(&node.value, 320))); + } + if !node.actions.is_empty() { + line.push_str(&format!(" [{}]", node.actions.join(","))); + } + if !node.enabled { + line.push_str(" disabled"); + } + if node.focused { + line.push_str(" focused"); + } + line +} + +fn display_string(value: &str, max_chars: usize) -> String { + let mut rendered = String::new(); + let mut chars = value.chars(); + for _ in 0..max_chars { + let Some(ch) = chars.next() else { break }; + match ch { + '\n' => rendered.push_str("\\n"), + '\r' => rendered.push_str("\\r"), + '\t' => rendered.push_str("\\t"), + '\\' => rendered.push_str("\\\\"), + '"' => rendered.push_str("\\\""), + other if other.is_control() => rendered.push(' '), + other => rendered.push(other), + } + } + if chars.next().is_some() { + rendered.push('…'); + } + rendered +} + +fn is_collapsible_container(node: &UiNode) -> bool { + !node.is_interactive() + && matches!( + canonical_role(&node.role).as_str(), + "group" | "unknown" | "layout_area" | "scroll_area" | "split_group" + ) + && node.title.is_empty() + && node.value.is_empty() +} + +fn descendant_count(node: &UiNode) -> usize { + node.children + .iter() + .map(|child| 1 + descendant_count(child)) + .sum() +} + +#[derive(Debug, Clone)] +pub struct SearchResult<'a> { + pub node: &'a UiNode, + pub score: u16, +} + +#[derive(Debug, Clone)] +pub struct SearchResults<'a> { + pub matches: Vec>, + pub total: usize, +} + +pub fn search<'a>(root: &'a UiNode, text: Option<&str>, role: Option<&str>) -> SearchResults<'a> { + let query = text.map(str::trim).filter(|value| !value.is_empty()); + let wanted_role = role.map(canonical_role); + let mut ranked = Vec::new(); + walk(root, &mut |node, path| { + if wanted_role + .as_ref() + .is_some_and(|wanted| canonical_role(&node.role) != *wanted) + { + return; + } + let score = match query { + Some(query) => match_score(node, query), + None => Some(1), + }; + if let Some(score) = score { + ranked.push((score, path.clone(), node)); + } + }); + ranked.sort_by(|left, right| right.0.cmp(&left.0).then_with(|| left.1.cmp(&right.1))); + let total = ranked.len(); + let matches = ranked + .into_iter() + .take(SEARCH_LIMIT) + .map(|(score, _, node)| SearchResult { node, score }) + .collect(); + SearchResults { matches, total } +} + +fn match_score(node: &UiNode, query: &str) -> Option { + let query = query.to_lowercase(); + [ + (&node.title, 30_u16), + (&node.value, 20), + (&node.description, 10), + ] + .into_iter() + .filter_map(|(candidate, field_bonus)| { + let candidate = candidate.trim().to_lowercase(); + if candidate.is_empty() { + return None; + } + let quality = if candidate == query { + 400 + } else if candidate.starts_with(&query) { + 300 + } else if candidate.contains(&query) { + 200 + } else if conservative_fuzzy(&candidate, &query) { + 100 + } else { + return None; + }; + Some(quality + field_bonus) + }) + .max() +} + +fn conservative_fuzzy(candidate: &str, query: &str) -> bool { + if query.chars().count() < 4 { + return false; + } + let candidate_word = candidate + .split(|ch: char| !ch.is_alphanumeric()) + .filter(|word| !word.is_empty()) + .min_by_key(|word| word.len().abs_diff(query.len())) + .unwrap_or(candidate); + let length = candidate_word.chars().count().max(query.chars().count()); + let allowed = if length >= 8 { 2 } else { 1 }; + adjacent_transposition(candidate_word, query) + || edit_distance_with_limit(candidate_word, query, allowed).is_some() +} + +fn adjacent_transposition(left: &str, right: &str) -> bool { + let left: Vec = left.chars().collect(); + let right: Vec = right.chars().collect(); + if left.len() != right.len() || left.len() < 2 { + return false; + } + let differences: Vec = left + .iter() + .zip(&right) + .enumerate() + .filter_map(|(index, (left, right))| (left != right).then_some(index)) + .collect(); + differences.len() == 2 + && differences[1] == differences[0] + 1 + && left[differences[0]] == right[differences[1]] + && left[differences[1]] == right[differences[0]] +} + +fn edit_distance_with_limit(left: &str, right: &str, limit: usize) -> Option { + let left: Vec = left.chars().collect(); + let right: Vec = right.chars().collect(); + if left.len().abs_diff(right.len()) > limit { + return None; + } + let mut previous: Vec = (0..=right.len()).collect(); + for (i, left_ch) in left.iter().enumerate() { + let mut current = vec![i + 1; right.len() + 1]; + let mut row_min = current[0]; + for (j, right_ch) in right.iter().enumerate() { + current[j + 1] = (previous[j + 1] + 1) + .min(current[j] + 1) + .min(previous[j] + usize::from(left_ch != right_ch)); + row_min = row_min.min(current[j + 1]); + } + if row_min > limit { + return None; + } + previous = current; + } + (previous[right.len()] <= limit).then_some(previous[right.len()]) +} + +#[derive(Debug, Clone)] +pub struct TreeDiff { + pub text: String, + pub confidence: f32, + pub use_full_view: bool, +} + +pub fn diff_trees(previous: &UiNode, successor: &UiNode) -> TreeDiff { + let old = flatten(previous); + let new = flatten(successor); + let root_changed = node_signature(previous) != node_signature(successor); + let mut used_old = HashSet::new(); + let mut matched: Vec<(&FlatNode<'_>, &FlatNode<'_>)> = Vec::new(); + let mut added: Vec<&FlatNode<'_>> = Vec::new(); + + for new_entry in &new { + let path_match = old.iter().enumerate().find(|(index, old_entry)| { + !used_old.contains(index) + && old_entry.path == new_entry.path + && old_entry.signature == new_entry.signature + }); + let signature_matches: Vec<_> = old + .iter() + .enumerate() + .filter(|(index, old_entry)| { + !used_old.contains(index) && old_entry.signature == new_entry.signature + }) + .collect(); + let candidate = + path_match.or_else(|| (signature_matches.len() == 1).then(|| signature_matches[0])); + if let Some((index, old_entry)) = candidate { + used_old.insert(index); + matched.push((old_entry, new_entry)); + } else { + added.push(new_entry); + } + } + let removed: Vec<&FlatNode<'_>> = old + .iter() + .enumerate() + .filter(|(index, _)| !used_old.contains(index)) + .map(|(_, entry)| entry) + .collect(); + let updated: Vec<(&FlatNode<'_>, &FlatNode<'_>)> = matched + .iter() + .copied() + .filter(|(old_entry, new_entry)| node_changed(old_entry.node, new_entry.node)) + .collect(); + let confidence = if old.len().max(new.len()) == 0 { + 1.0 + } else { + matched.len() as f32 / old.len().max(new.len()) as f32 + }; + let use_full_view = root_changed || confidence < 0.55; + + let mut lines = vec![format!( + "diff: +{} ~{} -{} (match confidence {:.0}%)", + added.len(), + updated.len(), + removed.len(), + confidence * 100.0 + )]; + for entry in added.iter().take(80) { + lines.push(format!("+ {}", render_line(entry.node, 0))); + } + for (old_entry, new_entry) in updated.iter().take(80) { + lines.push(format!( + "~ {} -> {}", + old_entry.ref_id, + render_line(new_entry.node, 0) + )); + } + for entry in removed.iter().take(80) { + lines.push(format!( + "- {} {} \"{}\"", + entry.ref_id, + canonical_role(&entry.node.role), + display_string(&entry.node.title, 160) + )); + } + let shown = added.len().min(80) + updated.len().min(80) + removed.len().min(80); + if shown < added.len() + updated.len() + removed.len() { + lines.push("… diff entries capped; inspect the successor state for details".into()); + } + TreeDiff { + text: lines.join("\n"), + confidence, + use_full_view, + } +} + +fn node_changed(left: &UiNode, right: &UiNode) -> bool { + left.value != right.value + || left.description != right.description + || left.frame != right.frame + || left.actions != right.actions + || left.enabled != right.enabled + || left.focused != right.focused +} + +struct FlatNode<'a> { + node: &'a UiNode, + ref_id: String, + path: Vec, + signature: (String, String), +} + +fn flatten(root: &UiNode) -> Vec> { + let mut entries = Vec::new(); + walk(root, &mut |node, path| { + entries.push(FlatNode { + node, + ref_id: node.ref_id.clone(), + path: path.clone(), + signature: node_signature(node), + }); + }); + entries +} + +fn node_signature(node: &UiNode) -> (String, String) { + (canonical_role(&node.role), node.title.trim().to_lowercase()) +} + +fn ref_number(ref_id: &str) -> Option { + ref_id.strip_prefix("@e")?.parse().ok() +} + +fn walk<'a>(root: &'a UiNode, callback: &mut impl FnMut(&'a UiNode, &Vec)) { + fn recurse<'a>( + node: &'a UiNode, + path: &mut Vec, + callback: &mut impl FnMut(&'a UiNode, &Vec), + ) { + callback(node, path); + for (index, child) in node.children.iter().enumerate() { + path.push(index); + recurse(child, path, callback); + path.pop(); + } + } + recurse(root, &mut Vec::new(), callback); +} + +fn walk_mut(root: &mut UiNode, callback: &mut impl FnMut(&mut UiNode, &Vec)) { + fn recurse( + node: &mut UiNode, + path: &mut Vec, + callback: &mut impl FnMut(&mut UiNode, &Vec), + ) { + callback(node, path); + for (index, child) in node.children.iter_mut().enumerate() { + path.push(index); + recurse(child, path, callback); + path.pop(); + } + } + recurse(root, &mut Vec::new(), callback); +} + +pub fn output_exceeds_limit(text: &str) -> bool { + text.len() > MAX_MODEL_BYTES || text.lines().count() > MAX_MODEL_LINES +} + +pub fn safe_prefix(text: &str, byte_limit: usize, line_limit: usize) -> (&str, usize) { + let mut end = text.len().min(byte_limit); + while !text.is_char_boundary(end) { + end -= 1; + } + if line_limit != usize::MAX { + let mut line_count = 0; + for (index, byte) in text[..end].bytes().enumerate() { + if byte == b'\n' { + line_count += 1; + if line_count >= line_limit { + end = index + 1; + break; + } + } + } + } + (&text[..end], end) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn node(role: &str, title: &str, children: Vec) -> UiNode { + UiNode { + role: role.into(), + title: title.into(), + enabled: true, + children, + ..UiNode::default() + } + } + + #[test] + fn folding_collapses_single_child_groups_but_keeps_interactive_nodes() { + let mut tree = node( + "window", + "Document", + vec![node( + "group", + "", + vec![node("group", "", vec![node("button", "Save", Vec::new())])], + )], + ); + tree.children[0].children[0].children[0].actions = vec!["press".into()]; + assign_refs(&mut tree); + let rendered = render_folded(&tree); + assert!(rendered.contains("@e4 button \"Save\" [press]")); + assert!(!rendered.contains("@e2 group")); + assert!(!rendered.contains("@e3 group")); + } + + #[test] + fn search_ranks_exact_prefix_substring_then_fuzzy() { + let mut tree = node( + "window", + "Root", + vec![ + node("button", "Save", Vec::new()), + node("button", "Save As", Vec::new()), + node("button", "Auto Save Settings", Vec::new()), + node("button", "Svae", Vec::new()), + node("link", "Save", Vec::new()), + ], + ); + assign_refs(&mut tree); + let results = search(&tree, Some("save"), Some("button")); + let titles: Vec<_> = results + .matches + .iter() + .map(|result| result.node.title.as_str()) + .collect(); + assert_eq!(titles, ["Save", "Save As", "Auto Save Settings", "Svae"]); + assert_eq!(results.total, 4); + } + + #[test] + fn successor_refs_and_diff_are_stable() { + let mut old = node("window", "Doc", vec![node("button", "Save", Vec::new())]); + assign_refs(&mut old); + let mut new = old.clone(); + new.children[0].value = "done".into(); + new.children.push(node("checkbox", "Autosave", Vec::new())); + assign_refs_from_previous(&old, &mut new); + assert_eq!(new.children[0].ref_id, old.children[0].ref_id); + let diff = diff_trees(&old, &new); + assert!(diff.text.contains("+1 ~1 -0")); + assert!(!diff.use_full_view); + } + + #[test] + fn safe_prefix_ends_on_utf8_and_line_boundaries() { + let text = "a\nβ\ncharlie"; + let (prefix, offset) = safe_prefix(text, 5, 2); + assert_eq!(prefix, "a\nβ\n"); + assert_eq!(offset, prefix.len()); + } +} diff --git a/crates/computer-use-mcp/src/permissions.rs b/crates/computer-use-mcp/src/permissions.rs new file mode 100644 index 00000000..fde71fa4 --- /dev/null +++ b/crates/computer-use-mcp/src/permissions.rs @@ -0,0 +1,170 @@ +//! macOS TCC permission checks and requests, shared by the computer-use +//! backend and the Settings → Computer Use / Browser pages. +//! +//! tcode is itself the signed `.app` the grants attach to, so there is no +//! helper-app attribution to worry about. Screen Recording grants only take +//! effect after the app restarts (macOS shows its own "Quit & Reopen" dialog); +//! callers must persist any restart-continuity marker *before* requesting. + +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PermissionKind { + Accessibility, + ScreenRecording, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct PermissionStatus { + pub accessibility: bool, + pub screen_recording: bool, +} + +impl PermissionStatus { + pub fn granted(&self, kind: PermissionKind) -> bool { + match kind { + PermissionKind::Accessibility => self.accessibility, + PermissionKind::ScreenRecording => self.screen_recording, + } + } + + pub fn all_granted(&self) -> bool { + self.accessibility && self.screen_recording + } +} + +/// Non-prompting snapshot of both TCC grants for this process. +pub fn check() -> PermissionStatus { + imp::check() +} + +/// Fire the OS prompt for one permission kind. Accessibility prompts inline; +/// Screen Recording prompts at most once per TCC reset, after which the user +/// must flip the toggle in System Settings — pair this with +/// [`open_settings_pane`]. Returns the (possibly already-granted) status. +pub fn request(kind: PermissionKind) -> bool { + imp::request(kind) +} + +/// Deep-link System Settings to the Privacy & Security pane for `kind`. +pub fn open_settings_pane(kind: PermissionKind) { + imp::open_settings_pane(kind) +} + +/// Start a fresh instance of tcode and return; the caller is responsible for +/// quitting the current instance afterwards. Prefers relaunching the enclosing +/// `.app` bundle (so LaunchServices identity — and thus TCC attribution — is +/// preserved); falls back to re-spawning the bare executable in dev builds. +pub fn relaunch_app() -> std::io::Result<()> { + let exe = std::env::current_exe()?; + let bundle = exe + .ancestors() + .find(|p| p.extension().is_some_and(|e| e == "app")) + .map(std::path::Path::to_path_buf); + match bundle { + Some(app) => { + tcode_services::process::command("open") + .arg("-n") + .arg(app) + .spawn()?; + } + None => { + tcode_services::process::command(exe).spawn()?; + } + } + Ok(()) +} + +#[cfg(target_os = "macos")] +mod imp { + use super::{PermissionKind, PermissionStatus}; + use core_foundation::base::TCFType; + use core_foundation::boolean::CFBoolean; + use core_foundation::dictionary::{CFDictionary, CFDictionaryRef}; + use core_foundation::string::{CFString, CFStringRef}; + + #[link(name = "ApplicationServices", kind = "framework")] + unsafe extern "C" { + fn AXIsProcessTrusted() -> bool; + fn AXIsProcessTrustedWithOptions(options: CFDictionaryRef) -> bool; + static kAXTrustedCheckOptionPrompt: CFStringRef; + } + + #[link(name = "CoreGraphics", kind = "framework")] + unsafe extern "C" { + fn CGPreflightScreenCaptureAccess() -> bool; + fn CGRequestScreenCaptureAccess() -> bool; + } + + pub(super) fn check() -> PermissionStatus { + PermissionStatus { + accessibility: unsafe { AXIsProcessTrusted() }, + screen_recording: unsafe { CGPreflightScreenCaptureAccess() }, + } + } + + pub(super) fn request(kind: PermissionKind) -> bool { + match kind { + PermissionKind::Accessibility => unsafe { + let key = CFString::wrap_under_get_rule(kAXTrustedCheckOptionPrompt); + let options = CFDictionary::from_CFType_pairs(&[( + key.as_CFType(), + CFBoolean::true_value().as_CFType(), + )]); + AXIsProcessTrustedWithOptions(options.as_concrete_TypeRef()) + }, + PermissionKind::ScreenRecording => unsafe { CGRequestScreenCaptureAccess() }, + } + } + + pub(super) fn open_settings_pane(kind: PermissionKind) { + let pane = match kind { + PermissionKind::Accessibility => "Privacy_Accessibility", + PermissionKind::ScreenRecording => "Privacy_ScreenCapture", + }; + let url = format!("x-apple.systempreferences:com.apple.preference.security?{pane}"); + let _ = tcode_services::process::command("open").arg(url).spawn(); + } +} + +#[cfg(not(target_os = "macos"))] +mod imp { + use super::{PermissionKind, PermissionStatus}; + + pub(super) fn check() -> PermissionStatus { + // Non-macOS platforms have no TCC; the backend is a stub there and the + // settings UI shows the platform as unsupported rather than ungranted. + PermissionStatus::default() + } + + pub(super) fn request(_kind: PermissionKind) -> bool { + false + } + + pub(super) fn open_settings_pane(_kind: PermissionKind) {} +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn status_granted_maps_kinds() { + let status = PermissionStatus { + accessibility: true, + screen_recording: false, + }; + assert!(status.granted(PermissionKind::Accessibility)); + assert!(!status.granted(PermissionKind::ScreenRecording)); + assert!(!status.all_granted()); + } + + #[test] + fn kind_serializes_snake_case() { + assert_eq!( + serde_json::to_string(&PermissionKind::ScreenRecording).unwrap(), + "\"screen_recording\"" + ); + } +} diff --git a/crates/computer-use-mcp/src/state.rs b/crates/computer-use-mcp/src/state.rs new file mode 100644 index 00000000..ab854365 --- /dev/null +++ b/crates/computer-use-mcp/src/state.rs @@ -0,0 +1,430 @@ +//! Bounded immutable observations and independently bounded output pages. + +use std::collections::{HashMap, VecDeque}; +use std::fmt; +use std::sync::{Arc, Mutex, OnceLock}; + +use crate::backend::RootInfo; +use crate::outline::{ + MAX_MODEL_LINES, PAGE_BYTES, PREVIEW_BYTES, UiNode, assign_refs, assign_refs_from_previous, + output_exceeds_limit, safe_prefix, +}; + +pub const OBSERVATION_CAPACITY: usize = 8; +pub const OUTPUT_CAPACITY: usize = 32; + +#[derive(Debug)] +pub struct Observation { + pub state_id: String, + pub root: RootInfo, + pub root_epoch: u64, + pub tree: UiNode, + pub screenshot_png: Option>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum StateError { + Evicted(String), + Stale { + state_id: String, + state_epoch: u64, + current_epoch: u64, + }, + UnknownElement(String), + UnknownOutput(String), + OutputOwnerMismatch { + output_ref: String, + expected: String, + actual: Option, + }, + InvalidOffset(usize), +} + +impl fmt::Display for StateError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Evicted(state_id) => write!( + formatter, + "state {state_id} was evicted from the observation cache; call observe_ui again" + ), + Self::Stale { + state_id, + state_epoch, + current_epoch, + } => write!( + formatter, + "state {state_id} is stale (root epoch {state_epoch}, current {current_epoch}); call observe_ui again" + ), + Self::UnknownElement(ref_id) => write!( + formatter, + "element ref {ref_id} is not owned by this state; call observe_ui again if the UI changed" + ), + Self::UnknownOutput(output_ref) => write!( + formatter, + "output continuation {output_ref} was evicted or does not exist; rerun the originating tool" + ), + Self::OutputOwnerMismatch { + output_ref, + expected, + actual, + } => write!( + formatter, + "output continuation {output_ref} belongs to state {}, not {expected}", + actual.as_deref().unwrap_or("") + ), + Self::InvalidOffset(offset) => write!( + formatter, + "byte offset {offset} is outside the text or splits a UTF-8 character" + ), + } + } +} + +impl std::error::Error for StateError {} + +#[derive(Debug, Clone)] +struct OutputEntry { + owner_state: Option, + text: Arc, + initial_offset: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OutputPage { + pub output_ref: String, + pub owner_state: Option, + pub text: String, + pub offset: usize, + pub next_offset: usize, + pub total_bytes: usize, + pub eof: bool, +} + +#[derive(Debug)] +pub struct StateStore { + observation_capacity: usize, + output_capacity: usize, + next_state: u64, + next_output: u64, + observations: HashMap>, + observation_lru: VecDeque, + root_epochs: HashMap, + outputs: HashMap, + output_lru: VecDeque, +} + +impl Default for StateStore { + fn default() -> Self { + Self::new(OBSERVATION_CAPACITY, OUTPUT_CAPACITY) + } +} + +impl StateStore { + pub fn new(observation_capacity: usize, output_capacity: usize) -> Self { + assert!(observation_capacity > 0); + assert!(output_capacity > 0); + Self { + observation_capacity, + output_capacity, + next_state: 1, + next_output: 1, + observations: HashMap::new(), + observation_lru: VecDeque::new(), + root_epochs: HashMap::new(), + outputs: HashMap::new(), + output_lru: VecDeque::new(), + } + } + + pub fn insert_observation( + &mut self, + root: RootInfo, + mut tree: UiNode, + screenshot_png: Option>, + ) -> Arc { + let identity = root.identity(); + let previous = self + .observation_lru + .iter() + .rev() + .filter_map(|state_id| self.observations.get(state_id)) + .find(|observation| observation.root.identity() == identity) + .cloned(); + if let Some(previous) = previous { + assign_refs_from_previous(&previous.tree, &mut tree); + } else { + assign_refs(&mut tree); + } + + let epoch = self.root_epochs.entry(identity).or_default(); + *epoch += 1; + let state_id = format!("S{}", self.next_state); + self.next_state += 1; + let observation = Arc::new(Observation { + state_id: state_id.clone(), + root, + root_epoch: *epoch, + tree, + screenshot_png, + }); + self.observations + .insert(state_id.clone(), Arc::clone(&observation)); + touch(&mut self.observation_lru, &state_id); + while self.observations.len() > self.observation_capacity { + if let Some(evicted) = self.observation_lru.pop_front() { + self.observations.remove(&evicted); + } + } + observation + } + + pub fn get(&mut self, state_id: &str) -> Result, StateError> { + let observation = self + .observations + .get(state_id) + .cloned() + .ok_or_else(|| StateError::Evicted(state_id.to_string()))?; + touch(&mut self.observation_lru, state_id); + Ok(observation) + } + + pub fn validate_for_action(&mut self, state_id: &str) -> Result, StateError> { + let observation = self.get(state_id)?; + let current_epoch = self + .root_epochs + .get(&observation.root.identity()) + .copied() + .unwrap_or_default(); + if current_epoch != observation.root_epoch { + return Err(StateError::Stale { + state_id: state_id.to_string(), + state_epoch: observation.root_epoch, + current_epoch, + }); + } + Ok(observation) + } + + pub fn register_output( + &mut self, + owner_state: Option<&str>, + text: impl Into>, + initial_offset: usize, + ) -> String { + let output_ref = format!("@o{}", self.next_output); + self.next_output += 1; + self.outputs.insert( + output_ref.clone(), + OutputEntry { + owner_state: owner_state.map(str::to_owned), + text: text.into(), + initial_offset, + }, + ); + touch(&mut self.output_lru, &output_ref); + while self.outputs.len() > self.output_capacity { + if let Some(evicted) = self.output_lru.pop_front() { + self.outputs.remove(&evicted); + } + } + output_ref + } + + pub fn bound_model_text(&mut self, owner_state: Option<&str>, text: String) -> String { + if !output_exceeds_limit(&text) { + return text; + } + let (preview, offset) = safe_prefix(&text, PREVIEW_BYTES, MAX_MODEL_LINES); + let preview = preview.to_string(); + let output_ref = self.register_output(owner_state, Arc::::from(text), offset); + format!( + "{preview}\n\n[output truncated: use read_text with ref {output_ref} and offset {offset}]" + ) + } + + pub fn read_output( + &mut self, + output_ref: &str, + state_id: Option<&str>, + offset: Option, + ) -> Result { + let entry = self + .outputs + .get(output_ref) + .cloned() + .ok_or_else(|| StateError::UnknownOutput(output_ref.to_string()))?; + if let Some(expected) = state_id + && entry.owner_state.as_deref() != Some(expected) + { + return Err(StateError::OutputOwnerMismatch { + output_ref: output_ref.to_string(), + expected: expected.to_string(), + actual: entry.owner_state, + }); + } + touch(&mut self.output_lru, output_ref); + page( + output_ref, + entry.owner_state, + &entry.text, + offset.unwrap_or(entry.initial_offset), + ) + } + + pub fn page_element_text( + &mut self, + state_id: &str, + ref_id: &str, + offset: usize, + ) -> Result { + let observation = self.get(state_id)?; + let node = observation + .tree + .find(ref_id) + .ok_or_else(|| StateError::UnknownElement(ref_id.to_string()))?; + let text: Arc = Arc::from(node.text()); + if text.len().saturating_sub(offset) > PAGE_BYTES { + let output_ref = self.register_output(Some(state_id), Arc::clone(&text), offset); + self.read_output(&output_ref, Some(state_id), Some(offset)) + } else { + page(ref_id, Some(state_id.to_string()), &text, offset) + } + } + + #[cfg(test)] + fn contains(&self, state_id: &str) -> bool { + self.observations.contains_key(state_id) + } +} + +fn page( + output_ref: &str, + owner_state: Option, + text: &str, + offset: usize, +) -> Result { + if offset > text.len() || !text.is_char_boundary(offset) { + return Err(StateError::InvalidOffset(offset)); + } + let remaining = &text[offset..]; + let (content, consumed) = safe_prefix(remaining, PAGE_BYTES, MAX_MODEL_LINES); + let next_offset = offset + consumed; + Ok(OutputPage { + output_ref: output_ref.to_string(), + owner_state, + text: content.to_string(), + offset, + next_offset, + total_bytes: text.len(), + eof: next_offset == text.len(), + }) +} + +fn touch(lru: &mut VecDeque, key: &str) { + if let Some(index) = lru.iter().position(|candidate| candidate == key) { + lru.remove(index); + } + lru.push_back(key.to_string()); +} + +static STORE: OnceLock> = OnceLock::new(); + +pub fn global() -> &'static Mutex { + STORE.get_or_init(|| Mutex::new(StateStore::default())) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::outline::Frame; + + fn root(window_id: u32) -> RootInfo { + RootInfo { + app_name: "Test".into(), + pid: 42, + title: format!("Window {window_id}"), + window_id, + frame: Frame { + x: 0.0, + y: 0.0, + w: 100.0, + h: 100.0, + }, + ..RootInfo::default() + } + } + + fn tree(title: &str) -> UiNode { + UiNode { + role: "window".into(), + title: title.into(), + enabled: true, + ..UiNode::default() + } + } + + #[test] + fn lru_evicts_least_recently_used_observation() { + let mut store = StateStore::new(2, 4); + let first = store.insert_observation(root(1), tree("one"), None); + let second = store.insert_observation(root(2), tree("two"), None); + store.get(&first.state_id).unwrap(); + let third = store.insert_observation(root(3), tree("three"), None); + assert!(store.contains(&first.state_id)); + assert!(!store.contains(&second.state_id)); + assert!(store.contains(&third.state_id)); + assert!(matches!( + store.get(&second.state_id), + Err(StateError::Evicted(_)) + )); + } + + #[test] + fn newer_root_epoch_rejects_actions_from_old_state() { + let mut store = StateStore::new(8, 4); + let first = store.insert_observation(root(1), tree("one"), None); + let second = store.insert_observation(root(1), tree("one changed"), None); + assert!(matches!( + store.validate_for_action(&first.state_id), + Err(StateError::Stale { .. }) + )); + assert!(store.validate_for_action(&second.state_id).is_ok()); + } + + #[test] + fn bounded_output_continuation_round_trips_without_mutation() { + let mut store = StateStore::new(2, 4); + let original = "0123456789abcdef\n".repeat(4_000); + let visible = store.bound_model_text(Some("S9"), original.clone()); + assert!(visible.len() < 20 * 1024); + let output_ref = visible + .split_whitespace() + .find(|part| part.starts_with("@o")) + .unwrap(); + let mut rebuilt = visible + .split("\n\n[output truncated") + .next() + .unwrap() + .to_string(); + let mut offset = rebuilt.len(); + loop { + let page = store + .read_output(output_ref, Some("S9"), Some(offset)) + .unwrap(); + rebuilt.push_str(&page.text); + if page.eof { + break; + } + offset = page.next_offset; + } + assert_eq!(rebuilt, original); + + let repeated = store + .read_output(output_ref, Some("S9"), Some(offset)) + .unwrap(); + let repeated_again = store + .read_output(output_ref, Some("S9"), Some(offset)) + .unwrap(); + assert_eq!(repeated, repeated_again); + } +} diff --git a/crates/computer-use-mcp/src/tools.rs b/crates/computer-use-mcp/src/tools.rs new file mode 100644 index 00000000..67c12b0e --- /dev/null +++ b/crates/computer-use-mcp/src/tools.rs @@ -0,0 +1,1612 @@ +use std::collections::HashMap; +use std::sync::{Arc, RwLock}; + +use axum::Router; +use axum::extract::State; +use axum::http::{StatusCode, header::AUTHORIZATION}; +use axum::response::{IntoResponse, Response}; +use axum::routing::any; +use rmcp::handler::server::router::tool::ToolRouter; +use rmcp::handler::server::wrapper::Parameters; +use rmcp::model::{ + CallToolResult, Content, Implementation, ProtocolVersion, ServerCapabilities, ServerInfo, +}; +use rmcp::transport::streamable_http_server::session::local::LocalSessionManager; +use rmcp::transport::streamable_http_server::{StreamableHttpServerConfig, StreamableHttpService}; +use rmcp::{ErrorData, ServerHandler, tool, tool_handler, tool_router}; +use serde::Deserialize; + +/// Kind of desktop root to match. +#[derive(Debug, Clone, Copy, Deserialize, schemars::JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum RootKind { + Window, + Dialog, + Sheet, + Menu, + Popover, +} + +/// Observation source to use for a desktop root. +#[derive(Debug, Clone, Copy, Deserialize, schemars::JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum ObserveMode { + Semantic, + Visual, + Fused, +} + +/// Input action to apply to a state-scoped UI element or screen coordinate. +#[derive(Debug, Clone, Deserialize, schemars::JsonSchema)] +pub struct UiAction { + /// Action to perform: press, click, set_text, type_text, keypress, scroll, drag, or move_mouse. + pub action: UiActionKind, + /// State-scoped element reference to target, when the action targets an element. + #[serde(default, rename = "ref")] + pub r#ref: Option, + /// Absolute screen x-coordinate, when targeting a coordinate. + #[serde(default)] + pub x: Option, + /// Absolute screen y-coordinate, when targeting a coordinate. + #[serde(default)] + pub y: Option, + /// Text to set or type for text-entry actions. + #[serde(default)] + pub text: Option, + /// Key names or chord components for a keypress action. + #[serde(default)] + pub keys: Option>, + /// Horizontal scroll delta for a scroll action. + #[serde(default)] + pub scroll_x: Option, + /// Vertical scroll delta for a scroll action. + #[serde(default)] + pub scroll_y: Option, + /// Absolute screen points describing a drag path. + #[serde(default)] + pub path: Option>, + /// Mouse button to use for click or drag actions. + #[serde(default)] + pub button: Option, + /// Number of clicks to issue for a click action. + #[serde(default)] + pub click_count: Option, +} + +/// Supported desktop input action. +#[derive(Debug, Clone, Copy, Deserialize, schemars::JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum UiActionKind { + Press, + Click, + SetText, + TypeText, + Keypress, + Scroll, + Drag, + MoveMouse, +} + +/// Mouse button used by pointer actions. +#[derive(Debug, Clone, Copy, Deserialize, schemars::JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum MouseButton { + Left, + Right, + Middle, +} + +/// Condition evaluated against a UI state. +#[derive(Debug, Clone, Deserialize, schemars::JsonSchema)] +pub struct UiCondition { + /// State-scoped element reference the condition should match. + #[serde(default, rename = "ref")] + pub r#ref: Option, + /// State-scoped ancestor reference that limits the search scope. + #[serde(default)] + pub scope_ref: Option, + /// Text the matching element should contain. + #[serde(default)] + pub text: Option, + /// Accessibility role the matching element should have. + #[serde(default)] + pub role: Option, + /// Accessibility value the matching element should have. + #[serde(default)] + pub value: Option, + /// Whether the matching element must become present or absent. + #[serde(default)] + pub until: Option, + /// Maximum time to wait for the condition, in milliseconds. + #[serde(default)] + pub timeout_ms: Option, +} + +/// Desired presence state for a UI condition. +#[derive(Debug, Clone, Copy, Deserialize, schemars::JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum ConditionUntil { + Present, + Absent, +} + +#[derive(Debug, Deserialize, schemars::JsonSchema)] +struct FindRootsParams { + /// Text to match against application and window titles. + #[serde(default)] + text: Option, + /// Application name to match. + #[serde(default)] + app: Option, + /// Application bundle identifier to match. + #[serde(default)] + bundle_id: Option, + /// Process identifier to match. + #[serde(default)] + pid: Option, + /// Desktop root kind to match. + #[serde(default)] + kind: Option, +} + +#[derive(Debug, Deserialize, schemars::JsonSchema)] +struct ObserveUiParams { + /// Desktop root reference to observe; omitted selects the frontmost root. + #[serde(default)] + root: Option, + /// Observation source: semantic, visual, or fused. + #[serde(default)] + mode: Option, +} + +#[derive(Debug, Deserialize, schemars::JsonSchema)] +struct SearchUiParams { + /// Identifier of the cached UI state to search. + state_id: String, + /// Text to rank against element names, values, and descriptions. + #[serde(default)] + text: Option, + /// Accessibility role to match. + #[serde(default)] + role: Option, +} + +#[derive(Debug, Deserialize, schemars::JsonSchema)] +struct ExpandUiParams { + /// Identifier of the cached UI state containing the element. + state_id: String, + /// State-scoped element reference to expand. + #[serde(rename = "ref")] + r#ref: String, + /// Maximum descendant depth to include. + #[serde(default)] + depth: Option, +} + +#[derive(Debug, Deserialize, schemars::JsonSchema)] +struct InspectUiParams { + /// Identifier of the cached UI state containing the element. + state_id: String, + /// State-scoped element reference to inspect. + #[serde(rename = "ref")] + r#ref: String, +} + +#[derive(Debug, Deserialize, schemars::JsonSchema)] +struct ActUiParams { + /// Identifier of the cached UI state against which actions are resolved. + state_id: String, + /// Ordered actions to execute as one transaction. + actions: Vec, + /// Optional postcondition used to verify the transaction outcome. + #[serde(default)] + expect: Option, +} + +#[derive(Debug, Deserialize, schemars::JsonSchema)] +struct ReadTextParams { + /// Identifier of the cached UI state containing the text; omitted uses the owning state encoded by the continuation. + #[serde(default)] + state_id: Option, + /// State-scoped element or continuation reference whose text should be read. + #[serde(rename = "ref")] + r#ref: String, + /// Byte offset at which to continue reading. + #[serde(default)] + offset: Option, +} + +#[derive(Debug, Deserialize, schemars::JsonSchema)] +struct WaitForParams { + /// Identifier of the cached UI state that scopes the condition. + state_id: String, + /// Condition fields to wait for. + #[serde(flatten)] + condition: UiCondition, +} + +#[derive(Clone)] +pub struct ComputerUseTools { + tool_router: ToolRouter, +} + +#[tool_router] +impl ComputerUseTools { + fn new() -> Self { + Self { + tool_router: Self::tool_router(), + } + } + + #[tool( + description = "Find and rank desktop window roots, returning state-scoped @rN references." + )] + async fn find_roots( + &self, + Parameters(params): Parameters, + ) -> Result { + Ok(dispatch::find_roots(params).await) + } + + #[tool( + description = "Observe a desktop root and return a folded outline, state_id, and screenshot when requested by the observation mode." + )] + async fn observe_ui( + &self, + Parameters(params): Parameters, + ) -> Result { + Ok(dispatch::observe_ui(params).await) + } + + #[tool( + description = "Search and rank elements in a cached UI state by text and accessibility role." + )] + async fn search_ui( + &self, + Parameters(params): Parameters, + ) -> Result { + Ok(dispatch::search_ui(params).await) + } + + #[tool(description = "Expand local outline context around a state-scoped element reference.")] + async fn expand_ui( + &self, + Parameters(params): Parameters, + ) -> Result { + Ok(dispatch::expand_ui(params).await) + } + + #[tool( + description = "Inspect an element's full accessibility attributes, frame, and supported actions." + )] + async fn inspect_ui( + &self, + Parameters(params): Parameters, + ) -> Result { + Ok(dispatch::inspect_ui(params).await) + } + + #[tool( + description = "Execute a transaction of desktop input actions against a cached UI state, optionally verifying a postcondition." + )] + async fn act_ui( + &self, + Parameters(params): Parameters, + ) -> Result { + Ok(dispatch::act_ui(params).await) + } + + #[tool(description = "Read a bounded page of long text owned by a state-scoped reference.")] + async fn read_text( + &self, + Parameters(params): Parameters, + ) -> Result { + Ok(dispatch::read_text(params).await) + } + + #[tool( + description = "Wait for a text, role, value, or referenced UI element to become present or absent." + )] + async fn wait_for( + &self, + Parameters(params): Parameters, + ) -> Result { + Ok(dispatch::wait_for(params).await) + } +} + +#[tool_handler] +impl ServerHandler for ComputerUseTools { + fn get_info(&self) -> ServerInfo { + ServerInfo { + protocol_version: ProtocolVersion::LATEST, + capabilities: ServerCapabilities::builder().enable_tools().build(), + server_info: Implementation::from_build_env(), + instructions: Some( + "Observe and control desktop applications through state-scoped accessibility references." + .into(), + ), + } + } +} + +pub type Service = StreamableHttpService; +pub type Services = HashMap; + +pub fn service() -> Service { + StreamableHttpService::new( + || Ok(ComputerUseTools::new()), + Arc::new(LocalSessionManager::default()), + StreamableHttpServerConfig::default(), + ) +} + +pub async fn serve( + listener: std::net::TcpListener, + services: Arc>, +) -> std::io::Result<()> { + let app = Router::new() + .route("/mcp", any(handle)) + .with_state(services); + listener.set_nonblocking(true)?; + axum::serve(tokio::net::TcpListener::from_std(listener)?, app).await +} + +#[derive(Debug, serde::Serialize)] +pub struct SmokeStep { + pub name: String, + pub ok: bool, + pub detail: String, +} + +#[derive(Debug, serde::Serialize)] +pub struct SmokeVerdict { + pub steps: Vec, + pub ok: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub reason: Option, +} + +pub struct SmokeRun { + pub verdict: SmokeVerdict, + pub exit_code: i32, +} + +/// Scripted TextEdit pass used by `cu-smoke`; it invokes the same dispatch +/// functions as MCP calls, with one direct global Cmd+N bootstrap for +/// TextEdit's no-window launch state. +pub async fn run_smoke() -> SmokeRun { + dispatch::run_smoke().await +} + +/// Hidden CLI-harness entry points. These deliberately deserialize the same +/// parameter structs and call the same dispatch functions as the MCP router. +pub async fn debug_find_roots(filter_json: &str) -> Result { + let params = if filter_json.is_empty() { + serde_json::from_str("{}")? + } else { + serde_json::from_str(filter_json)? + }; + Ok(dispatch::find_roots(params).await) +} + +pub async fn debug_observe(root: &str) -> CallToolResult { + dispatch::observe_ui(ObserveUiParams { + root: (!root.is_empty()).then(|| root.to_string()), + mode: Some(ObserveMode::Semantic), + }) + .await +} + +pub async fn debug_search(state_id: &str, text: &str) -> CallToolResult { + dispatch::search_ui(SearchUiParams { + state_id: state_id.to_string(), + text: Some(text.to_string()), + role: None, + }) + .await +} + +pub async fn debug_act(params_json: &str) -> Result { + let params = serde_json::from_str(params_json)?; + Ok(dispatch::act_ui(params).await) +} + +pub struct DebugScreenshot { + pub result: CallToolResult, + pub png: Option>, +} + +pub async fn debug_screenshot(root: Option<&str>) -> DebugScreenshot { + let result = dispatch::observe_ui(ObserveUiParams { + root: root.filter(|root| !root.is_empty()).map(str::to_string), + mode: Some(ObserveMode::Visual), + }) + .await; + let png = result + .content + .iter() + .find_map(|content| content.as_image()) + .and_then(|image| { + base64::Engine::decode(&base64::engine::general_purpose::STANDARD, &image.data).ok() + }); + DebugScreenshot { result, png } +} + +async fn handle( + State(services): State>>, + req: axum::extract::Request, +) -> Response { + let token = req + .headers() + .get(AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.strip_prefix("Bearer ")); + let service = token.and_then(|token| services.read().unwrap().get(token).cloned()); + let Some(service) = service else { + return (StatusCode::UNAUTHORIZED, "unauthorized").into_response(); + }; + let response = service.handle(req).await; + let (parts, body) = response.into_parts(); + Response::from_parts(parts, axum::body::Body::new(body)) +} + +mod dispatch { + use super::*; + use std::collections::HashMap; + use std::sync::{Mutex, OnceLock}; + use std::time::{Duration, Instant}; + + use base64::Engine as _; + use serde_json::json; + + use crate::backend::{ + ActionKind as BackendActionKind, ActionOutcome, ActionRequest, ActionResult, Backend, + CapturePolicy, MouseButton as BackendMouseButton, ObserveRequest, RootFilters, RootInfo, + RootKind as BackendRootKind, RootObservation, + }; + use crate::outline::{self, UiNode}; + + const DEFAULT_TIMEOUT_MS: u64 = 3_000; + const MAX_TIMEOUT_MS: u64 = 30_000; + const POLL_INTERVAL_MS: u64 = 100; + + #[derive(Default)] + struct RootRegistry { + next_ref: u64, + by_identity: HashMap, + by_ref: HashMap, + } + + impl RootRegistry { + fn refresh(&mut self, roots: Vec) -> Vec { + if self.next_ref == 0 { + self.next_ref = 1; + } + roots + .into_iter() + .map(|mut root| { + let identity = root.identity(); + let ref_id = self + .by_identity + .entry(identity) + .or_insert_with(|| { + let ref_id = format!("@r{}", self.next_ref); + self.next_ref += 1; + ref_id + }) + .clone(); + root.ref_id.clone_from(&ref_id); + self.by_ref.insert(ref_id, root.clone()); + root + }) + .collect() + } + + fn get(&self, ref_id: &str) -> Option { + self.by_ref.get(ref_id).cloned() + } + } + + static ROOTS: OnceLock> = OnceLock::new(); + static OBSERVATION_TRANSACTION: OnceLock> = OnceLock::new(); + + fn roots() -> &'static Mutex { + ROOTS.get_or_init(|| Mutex::new(RootRegistry::default())) + } + + fn observation_transaction() -> &'static tokio::sync::Mutex<()> { + OBSERVATION_TRANSACTION.get_or_init(|| tokio::sync::Mutex::new(())) + } + + pub(super) async fn run_smoke() -> SmokeRun { + #[cfg(not(target_os = "macos"))] + { + SmokeRun { + verdict: SmokeVerdict { + steps: Vec::new(), + ok: false, + reason: Some("unsupported platform: computer use requires macOS".into()), + }, + exit_code: 2, + } + } + + #[cfg(target_os = "macos")] + { + let permissions = crate::permissions::check(); + if !permissions.all_granted() { + let mut missing = Vec::new(); + if !permissions.accessibility { + missing.push("accessibility"); + } + if !permissions.screen_recording { + missing.push("screen_recording"); + } + return SmokeRun { + verdict: SmokeVerdict { + steps: Vec::new(), + ok: false, + reason: Some(format!("missing permissions: {}", missing.join(", "))), + }, + exit_code: 2, + }; + } + + let mut steps = Vec::new(); + let open_status = tcode_services::process::command("open") + .arg("-a") + .arg("TextEdit") + .status(); + match open_status { + Ok(status) if status.success() => steps.push(SmokeStep { + name: "launch_textedit".into(), + ok: true, + detail: status.to_string(), + }), + Ok(status) => { + return smoke_failure( + steps, + "launch_textedit", + format!("open exited with {status}"), + 1, + ); + } + Err(error) => { + return smoke_failure( + steps, + "launch_textedit", + format!("failed to spawn open: {error}"), + 1, + ); + } + } + tokio::time::sleep(Duration::from_millis(700)).await; + + // TextEdit can launch with no document and therefore no root. A + // global Cmd+N is the only action that does not need a state ref. + let backend = crate::backend::platform_backend(); + let bootstrap = ActionRequest { + kind: BackendActionKind::Keypress, + target_path: None, + target_frame: None, + target_role: None, + target_title: None, + target_actions: Vec::new(), + x: None, + y: None, + text: None, + keys: Some(vec!["cmd+n".into()]), + scroll_x: None, + scroll_y: None, + path: None, + button: BackendMouseButton::Left, + click_count: 1, + }; + match backend.perform_action(&RootInfo::default(), &bootstrap) { + Ok(result) if result.outcome != ActionOutcome::Didnt => steps.push(SmokeStep { + name: "fresh_document".into(), + ok: true, + detail: result.message, + }), + Ok(result) => { + return smoke_failure(steps, "fresh_document", result.message, 1); + } + Err(error) => { + return smoke_failure(steps, "fresh_document", error.to_string(), 1); + } + } + tokio::time::sleep(Duration::from_millis(700)).await; + + let roots_result = find_roots(FindRootsParams { + text: None, + app: Some("TextEdit".into()), + bundle_id: None, + pid: None, + kind: Some(RootKind::Window), + }) + .await; + let roots_text = match successful_text(&roots_result) { + Ok(text) => text, + Err(error) => return smoke_failure(steps, "find_roots", error, 1), + }; + let root_ref = roots_text + .lines() + .find_map(|line| line.trim_start().strip_prefix("@r")) + .and_then(|suffix| suffix.split_whitespace().next()) + .map(|suffix| format!("@r{suffix}")); + let Some(root_ref) = root_ref else { + return smoke_failure( + steps, + "find_roots", + "no TextEdit window root was found".into(), + 1, + ); + }; + steps.push(SmokeStep { + name: "find_roots".into(), + ok: true, + detail: root_ref.clone(), + }); + + let observe_result = observe_ui(ObserveUiParams { + root: Some(root_ref), + mode: Some(ObserveMode::Semantic), + }) + .await; + let observe_text = match successful_text(&observe_result) { + Ok(text) => text, + Err(error) => return smoke_failure(steps, "observe_ui", error, 1), + }; + let Some(state_id) = extract_state_id(&observe_text) else { + return smoke_failure( + steps, + "observe_ui", + "observe_ui response had no state_id".into(), + 1, + ); + }; + let observation = match crate::state::global().lock().unwrap().get(&state_id) { + Ok(observation) => observation, + Err(error) => { + return smoke_failure(steps, "observe_ui", error.to_string(), 1); + } + }; + let Some(target_ref) = editable_ref(&observation.tree) else { + return smoke_failure( + steps, + "observe_ui", + "TextEdit tree contained no editable text element".into(), + 1, + ); + }; + steps.push(SmokeStep { + name: "observe_ui".into(), + ok: true, + detail: format!("state_id={state_id} target={target_ref}"), + }); + + let nonce = format!("tcode-cu-smoke-{}", uuid::Uuid::new_v4().simple()); + let act_result = act_ui(ActUiParams { + state_id, + actions: vec![UiAction { + action: UiActionKind::TypeText, + r#ref: Some(target_ref), + x: None, + y: None, + text: Some(nonce.clone()), + keys: None, + scroll_x: None, + scroll_y: None, + path: None, + button: None, + click_count: None, + }], + expect: Some(UiCondition { + r#ref: None, + scope_ref: None, + text: Some(nonce.clone()), + role: None, + value: None, + until: Some(ConditionUntil::Present), + timeout_ms: Some(5_000), + }), + }) + .await; + let act_text = match successful_text(&act_result) { + Ok(text) => text, + Err(error) => return smoke_failure(steps, "act_ui", error, 1), + }; + let Some(successor_id) = extract_state_id(&act_text) else { + return smoke_failure( + steps, + "act_ui", + "act_ui response had no successor state_id".into(), + 1, + ); + }; + if act_text.contains("\"outcome\": \"didnt\"") { + return smoke_failure(steps, "act_ui", act_text, 1); + } + steps.push(SmokeStep { + name: "act_ui".into(), + ok: true, + detail: format!("state_id={successor_id} nonce={nonce}"), + }); + + let wait_result = wait_for(WaitForParams { + state_id: successor_id, + condition: UiCondition { + r#ref: None, + scope_ref: None, + text: Some(nonce), + role: None, + value: None, + until: Some(ConditionUntil::Present), + timeout_ms: Some(3_000), + }, + }) + .await; + let wait_text = match successful_text(&wait_result) { + Ok(text) => text, + Err(error) => return smoke_failure(steps, "wait_for", error, 1), + }; + if !wait_text.contains("\"status\": \"matched\"") { + return smoke_failure(steps, "wait_for", wait_text, 1); + } + steps.push(SmokeStep { + name: "wait_for".into(), + ok: true, + detail: extract_state_id(&wait_text).unwrap_or_else(|| "matched".into()), + }); + SmokeRun { + verdict: SmokeVerdict { + steps, + ok: true, + reason: None, + }, + exit_code: 0, + } + } + } + + #[cfg(target_os = "macos")] + fn smoke_failure( + mut steps: Vec, + name: &str, + detail: String, + exit_code: i32, + ) -> SmokeRun { + steps.push(SmokeStep { + name: name.into(), + ok: false, + detail: detail.clone(), + }); + SmokeRun { + verdict: SmokeVerdict { + steps, + ok: false, + reason: Some(detail), + }, + exit_code, + } + } + + #[cfg(target_os = "macos")] + fn successful_text(result: &CallToolResult) -> Result { + let text = result + .content + .iter() + .find_map(|content| content.as_text()) + .map(|content| content.text.clone()) + .unwrap_or_else(|| "tool returned no text content".into()); + if result.is_error == Some(true) { + Err(text) + } else { + Ok(text) + } + } + + #[cfg(target_os = "macos")] + fn extract_state_id(text: &str) -> Option { + for line in text.lines() { + let line = line.trim(); + if let Some(state_id) = line.strip_prefix("state_id:") { + return Some(state_id.trim().to_string()); + } + if line.starts_with("\"state_id\"") { + return line.split('"').nth(3).map(str::to_string); + } + } + None + } + + #[cfg(target_os = "macos")] + fn editable_ref(tree: &UiNode) -> Option { + fn collect(node: &UiNode, candidates: &mut Vec<(bool, String)>) { + if matches!( + outline::canonical_role(&node.role).as_str(), + "text_area" | "text_field" | "search_field" + ) && node.enabled + { + candidates.push((node.focused, node.ref_id.clone())); + } + for child in &node.children { + collect(child, candidates); + } + } + let mut candidates = Vec::new(); + collect(tree, &mut candidates); + candidates.sort_by_key(|(focused, _)| !focused); + candidates.into_iter().next().map(|(_, ref_id)| ref_id) + } + + pub(super) async fn find_roots(params: FindRootsParams) -> CallToolResult { + let permissions = permissions(); + if let Some(result) = permission_gate(permissions, true, false) { + return result; + } + let backend = crate::backend::platform_backend(); + let filters = RootFilters { + text: params.text, + app: params.app, + bundle_id: params.bundle_id, + pid: params.pid, + kind: params.kind.map(root_kind), + }; + let discovered = match backend.list_roots(&filters) { + Ok(roots) => roots, + Err(error) => return backend_error(error), + }; + let roots = roots().lock().unwrap().refresh(discovered); + let mut lines = vec![format!("roots: {} (frontmost first)", roots.len())]; + for root in roots { + lines.push(format!( + "{} {} app=\"{}\" bundle_id=\"{}\" pid={} title=\"{}\" window_id={} frame=({:.0},{:.0},{:.0},{:.0})", + root.ref_id, + root.kind, + escaped(&root.app_name), + escaped(&root.bundle_id), + root.pid, + escaped(&root.title), + root.window_id, + root.frame.x, + root.frame.y, + root.frame.w, + root.frame.h + )); + } + bounded_success(None, lines.join("\n"), Vec::new()) + } + + pub(super) async fn observe_ui(params: ObserveUiParams) -> CallToolResult { + let permissions = permissions(); + let needs_accessibility = !matches!(params.mode, Some(ObserveMode::Visual)); + let needs_screen_recording = + matches!(params.mode, Some(ObserveMode::Visual | ObserveMode::Fused)); + if let Some(result) = + permission_gate(permissions, needs_accessibility, needs_screen_recording) + { + return result; + } + let config = crate::config::get(); + if config.image_mode == crate::config::ImageMode::Always + && let Some(result) = permission_gate(permissions, false, true) + { + return result; + } + let _transaction = observation_transaction().lock().await; + let backend = crate::backend::platform_backend(); + let root = match resolve_root(backend.as_ref(), params.root.as_deref()) { + Ok(root) => root, + Err(result) => return *result, + }; + let capture = capture_policy(config.image_mode, params.mode, &permissions); + let request = ObserveRequest { + semantic: !matches!(params.mode, Some(ObserveMode::Visual)), + capture, + }; + let observed = match backend.observe(&root, request) { + Ok(observed) => observed, + Err(error) => return backend_error(error), + }; + save_observation(observed, capture_warning(config.image_mode, &permissions)) + } + + pub(super) async fn search_ui(params: SearchUiParams) -> CallToolResult { + let permissions = permissions(); + if let Some(result) = permission_gate(permissions, true, false) { + return result; + } + let observation = match crate::state::global().lock().unwrap().get(¶ms.state_id) { + Ok(observation) => observation, + Err(error) => return tool_error(&error.to_string()), + }; + let results = outline::search( + &observation.tree, + params.text.as_deref(), + params.role.as_deref(), + ); + let mut lines = vec![format!( + "state_id: {}\nmatches: {} (showing {})", + observation.state_id, + results.total, + results.matches.len() + )]; + for result in results.matches { + lines.push(format!( + "score={} {}", + result.score, + outline::render_line(result.node, 0) + )); + } + bounded_success(Some(&observation.state_id), lines.join("\n"), Vec::new()) + } + + pub(super) async fn expand_ui(params: ExpandUiParams) -> CallToolResult { + let permissions = permissions(); + if let Some(result) = permission_gate(permissions, true, false) { + return result; + } + let observation = match crate::state::global().lock().unwrap().get(¶ms.state_id) { + Ok(observation) => observation, + Err(error) => return tool_error(&error.to_string()), + }; + let depth = params.depth.unwrap_or(3).min(12) as usize; + let expanded = match outline::render_expanded(&observation.tree, ¶ms.r#ref, depth) { + Ok(expanded) => expanded, + Err(error) => return tool_error(&error), + }; + bounded_success( + Some(&observation.state_id), + format!("state_id: {}\n{expanded}", observation.state_id), + Vec::new(), + ) + } + + pub(super) async fn inspect_ui(params: InspectUiParams) -> CallToolResult { + let permissions = permissions(); + if let Some(result) = permission_gate(permissions, true, false) { + return result; + } + let observation = match crate::state::global().lock().unwrap().get(¶ms.state_id) { + Ok(observation) => observation, + Err(error) => return tool_error(&error.to_string()), + }; + let Some(node) = observation.tree.find(¶ms.r#ref) else { + return tool_error(&crate::state::StateError::UnknownElement(params.r#ref).to_string()); + }; + let value = json!({ + "state_id": observation.state_id, + "ref": node.ref_id, + "role": node.role, + "title": node.title, + "value": node.value, + "description": node.description, + "frame": node.frame, + "actions": node.actions, + "enabled": node.enabled, + "focused": node.focused, + "child_count": node.children.len(), + }); + let text = serde_json::to_string_pretty(&value).unwrap_or_else(|_| value.to_string()); + bounded_success(Some(&observation.state_id), text, Vec::new()) + } + + pub(super) async fn act_ui(params: ActUiParams) -> CallToolResult { + let permissions = permissions(); + if let Some(result) = permission_gate(permissions, true, false) { + return result; + } + if !crate::config::get().allow_input { + return tool_error( + "observe-only mode is enabled in Settings → Computer Use; input actions are disabled", + ); + } + if params.actions.is_empty() { + return tool_error("act_ui requires at least one action"); + } + let _transaction = observation_transaction().lock().await; + let previous = match crate::state::global() + .lock() + .unwrap() + .validate_for_action(¶ms.state_id) + { + Ok(observation) => observation, + Err(error) => return tool_error(&error.to_string()), + }; + let backend = crate::backend::platform_backend(); + let mut step_results = Vec::new(); + let mut stopped_at = None; + for (index, action) in params.actions.iter().enumerate() { + let result = match prepare_action(&previous.tree, action) { + Ok(request) => backend + .perform_action(&previous.root, &request) + .unwrap_or_else(|error| ActionResult::didnt(error.to_string())), + Err(error) => ActionResult::didnt(error), + }; + let didnt = result.outcome == ActionOutcome::Didnt; + step_results.push(json!({ + "index": index + 1, + "action": action_name(action.action), + "outcome": result.outcome, + "message": result.message, + })); + if didnt { + stopped_at = Some(index + 1); + break; + } + } + + let expectation_preexisting = params + .expect + .as_ref() + .is_some_and(|condition| condition_satisfied(&previous.tree, condition)); + let (mut successor, expectation_status, root_changed) = match poll_successor( + backend.as_ref(), + &previous, + params.expect.as_ref(), + expectation_preexisting, + ) + .await + { + Ok(result) => result, + Err(error) => return backend_error(error), + }; + outline::assign_refs_from_previous(&previous.tree, &mut successor.tree); + let successor = crate::state::global().lock().unwrap().insert_observation( + successor.root, + successor.tree, + successor.screenshot_png, + ); + let diff = outline::diff_trees(&previous.tree, &successor.tree); + let expectation_failed = expectation_status == "failed"; + let any_unknown = step_results + .iter() + .any(|result| result["outcome"] == "unknown"); + let outcome = if stopped_at.is_some() || expectation_failed { + "didnt" + } else if params.expect.is_some() && expectation_status == "verified" { + "worked" + } else if any_unknown { + "unknown" + } else { + "worked" + }; + let report = json!({ + "state_id": successor.state_id, + "previous_state_id": previous.state_id, + "outcome": outcome, + "stopped_at": stopped_at, + "steps": step_results, + "expect": expectation_status, + "root_changed": root_changed, + "diff_confidence": diff.confidence, + }); + let mut text = serde_json::to_string_pretty(&report).unwrap_or_else(|_| report.to_string()); + text.push('\n'); + if diff.use_full_view || root_changed { + text.push_str("successor full view:\n"); + text.push_str(&outline::render_folded(&successor.tree)); + } else { + text.push_str(&diff.text); + } + bounded_success(Some(&successor.state_id), text, Vec::new()) + } + + pub(super) async fn read_text(params: ReadTextParams) -> CallToolResult { + let permissions = permissions(); + if let Some(result) = permission_gate(permissions, true, false) { + return result; + } + let offset = match params.offset.map(usize::try_from).transpose() { + Ok(offset) => offset, + Err(_) => return tool_error("offset is too large for this platform"), + }; + let page = if params.r#ref.starts_with("@o") { + crate::state::global().lock().unwrap().read_output( + ¶ms.r#ref, + params.state_id.as_deref(), + offset, + ) + } else if params.r#ref.starts_with("@e") { + let Some(state_id) = params.state_id.as_deref() else { + return tool_error("state_id is required when reading an @e element ref"); + }; + crate::state::global().lock().unwrap().page_element_text( + state_id, + ¶ms.r#ref, + offset.unwrap_or(0), + ) + } else { + return tool_error("read_text ref must be an @e element or @o continuation"); + }; + let page = match page { + Ok(page) => page, + Err(error) => return tool_error(&error.to_string()), + }; + let owner = page.owner_state.as_deref().unwrap_or("none"); + let continuation = if page.eof { + "eof".to_string() + } else { + format!( + "continue with ref {} offset {}", + page.output_ref, page.next_offset + ) + }; + bounded_success( + page.owner_state.as_deref(), + format!( + "ref: {}\nstate_id: {}\noffset: {}\nnext_offset: {}\ntotal_bytes: {}\neof: {}\n{}\n---\n{}", + page.output_ref, + owner, + page.offset, + page.next_offset, + page.total_bytes, + page.eof, + continuation, + page.text + ), + Vec::new(), + ) + } + + pub(super) async fn wait_for(params: WaitForParams) -> CallToolResult { + let permissions = permissions(); + if let Some(result) = permission_gate(permissions, true, false) { + return result; + } + let _transaction = observation_transaction().lock().await; + let previous = match crate::state::global() + .lock() + .unwrap() + .validate_for_action(¶ms.state_id) + { + Ok(observation) => observation, + Err(error) => return tool_error(&error.to_string()), + }; + let backend = crate::backend::platform_backend(); + let timeout = Duration::from_millis( + params + .condition + .timeout_ms + .unwrap_or(DEFAULT_TIMEOUT_MS) + .min(MAX_TIMEOUT_MS), + ); + let deadline = Instant::now() + timeout; + let mut polls = 0_u64; + let (mut observed, root_changed, matched) = loop { + polls += 1; + let (mut observed, root_changed) = match observe_with_root_fallback( + backend.as_ref(), + &previous.root, + ObserveRequest { + semantic: true, + capture: CapturePolicy::Never, + }, + ) { + Ok(observed) => observed, + Err(error) => return backend_error(error), + }; + outline::assign_refs_from_previous(&previous.tree, &mut observed.tree); + if condition_satisfied(&observed.tree, ¶ms.condition) { + break (observed, root_changed, true); + } + if Instant::now() >= deadline { + break (observed, root_changed, false); + } + tokio::time::sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; + }; + outline::assign_refs_from_previous(&previous.tree, &mut observed.tree); + let successor = crate::state::global().lock().unwrap().insert_observation( + observed.root, + observed.tree, + observed.screenshot_png, + ); + let status = if matched { "matched" } else { "timeout" }; + let report = json!({ + "state_id": successor.state_id, + "previous_state_id": previous.state_id, + "status": status, + "until": until_name(params.condition.until), + "polls": polls, + "root_changed": root_changed, + }); + let mut text = serde_json::to_string_pretty(&report).unwrap_or_else(|_| report.to_string()); + text.push('\n'); + text.push_str(&outline::render_folded(&successor.tree)); + bounded_success(Some(&successor.state_id), text, Vec::new()) + } + + #[cfg(target_os = "macos")] + type PermissionSnapshot = crate::permissions::PermissionStatus; + + #[cfg(not(target_os = "macos"))] + #[derive(Clone, Copy)] + struct PermissionSnapshot; + + #[cfg(target_os = "macos")] + fn permissions() -> PermissionSnapshot { + crate::permissions::check() + } + + #[cfg(not(target_os = "macos"))] + fn permissions() -> PermissionSnapshot { + PermissionSnapshot + } + + #[cfg(target_os = "macos")] + fn permission_gate( + permissions: PermissionSnapshot, + needs_accessibility: bool, + needs_screen_recording: bool, + ) -> Option { + if needs_accessibility && !permissions.accessibility { + return Some(tool_error( + "Accessibility permission is missing; grant it in tcode Settings → Computer Use.", + )); + } + if needs_screen_recording && !permissions.screen_recording { + return Some(tool_error( + "Screen Recording permission is missing; grant it in tcode Settings → Computer Use.", + )); + } + None + } + + #[cfg(not(target_os = "macos"))] + fn permission_gate( + _permissions: PermissionSnapshot, + _needs_accessibility: bool, + _needs_screen_recording: bool, + ) -> Option { + Some(backend_error(crate::backend::BackendError::unsupported())) + } + + fn root_kind(kind: RootKind) -> BackendRootKind { + match kind { + RootKind::Window => BackendRootKind::Window, + RootKind::Dialog => BackendRootKind::Dialog, + RootKind::Sheet => BackendRootKind::Sheet, + RootKind::Menu => BackendRootKind::Menu, + RootKind::Popover => BackendRootKind::Popover, + } + } + + fn resolve_root( + backend: &dyn Backend, + requested: Option<&str>, + ) -> Result> { + if let Some(requested) = requested + && let Some(root) = roots().lock().unwrap().get(requested) + { + return Ok(root); + } + let discovered = backend + .list_roots(&RootFilters::default()) + .map_err(|error| Box::new(backend_error(error)))?; + let discovered = roots().lock().unwrap().refresh(discovered); + match requested { + Some(requested) => discovered + .into_iter() + .find(|root| root.ref_id == requested) + .ok_or_else(|| { + Box::new(tool_error(&format!( + "root ref {requested} is no longer available; call find_roots again" + ))) + }), + None => discovered + .into_iter() + .next() + .ok_or_else(|| Box::new(tool_error("no on-screen desktop roots were found"))), + } + } + + fn capture_policy( + configured: crate::config::ImageMode, + requested: Option, + permissions: &PermissionSnapshot, + ) -> CapturePolicy { + #[cfg(not(target_os = "macos"))] + let _ = permissions; + if configured == crate::config::ImageMode::Never { + return CapturePolicy::Never; + } + if matches!(requested, Some(ObserveMode::Visual | ObserveMode::Fused)) + || configured == crate::config::ImageMode::Always + { + return CapturePolicy::Always; + } + #[cfg(target_os = "macos")] + if !permissions.screen_recording { + return CapturePolicy::Never; + } + CapturePolicy::IfSparse + } + + fn capture_warning( + configured: crate::config::ImageMode, + permissions: &PermissionSnapshot, + ) -> Option<&'static str> { + #[cfg(target_os = "macos")] + if configured == crate::config::ImageMode::Auto && !permissions.screen_recording { + return Some( + "screenshot omitted in auto mode because Screen Recording permission is missing", + ); + } + let _ = (configured, permissions); + None + } + + fn save_observation(observed: RootObservation, warning: Option<&str>) -> CallToolResult { + let screenshot_for_response = observed.screenshot_png.clone(); + let observation = crate::state::global().lock().unwrap().insert_observation( + observed.root, + observed.tree, + observed.screenshot_png, + ); + let mut text = format!( + "state_id: {}\nroot: {} app=\"{}\" title=\"{}\"\nelements: {} interactive: {}", + observation.state_id, + observation.root.ref_id, + escaped(&observation.root.app_name), + escaped(&observation.root.title), + count_nodes(&observation.tree), + outline::interactive_count(&observation.tree) + ); + if let Some(warning) = warning { + text.push_str("\nwarning: "); + text.push_str(warning); + } + text.push('\n'); + text.push_str(&outline::render_folded(&observation.tree)); + let extra = screenshot_for_response + .map(|png| { + Content::image( + base64::engine::general_purpose::STANDARD.encode(png), + "image/png", + ) + }) + .into_iter() + .collect(); + bounded_success(Some(&observation.state_id), text, extra) + } + + fn prepare_action(tree: &UiNode, action: &UiAction) -> Result { + let target = action.r#ref.as_deref().map(|ref_id| { + let node = tree.find(ref_id).ok_or_else(|| { + crate::state::StateError::UnknownElement(ref_id.to_string()).to_string() + })?; + let path = outline::path_to_ref(tree, ref_id).ok_or_else(|| { + crate::state::StateError::UnknownElement(ref_id.to_string()).to_string() + })?; + Ok::<_, String>((node, path)) + }); + let target = target.transpose()?; + Ok(ActionRequest { + kind: match action.action { + UiActionKind::Press => BackendActionKind::Press, + UiActionKind::Click => BackendActionKind::Click, + UiActionKind::SetText => BackendActionKind::SetText, + UiActionKind::TypeText => BackendActionKind::TypeText, + UiActionKind::Keypress => BackendActionKind::Keypress, + UiActionKind::Scroll => BackendActionKind::Scroll, + UiActionKind::Drag => BackendActionKind::Drag, + UiActionKind::MoveMouse => BackendActionKind::MoveMouse, + }, + target_path: target.as_ref().map(|(_, path)| path.clone()), + target_frame: target.as_ref().map(|(node, _)| node.frame), + target_role: target.as_ref().map(|(node, _)| node.role.clone()), + target_title: target.as_ref().map(|(node, _)| node.title.clone()), + target_actions: target + .as_ref() + .map(|(node, _)| node.actions.clone()) + .unwrap_or_default(), + x: action.x, + y: action.y, + text: action.text.clone(), + keys: action.keys.clone(), + scroll_x: action.scroll_x, + scroll_y: action.scroll_y, + path: action.path.clone(), + button: match action.button.unwrap_or(MouseButton::Left) { + MouseButton::Left => BackendMouseButton::Left, + MouseButton::Right => BackendMouseButton::Right, + MouseButton::Middle => BackendMouseButton::Middle, + }, + click_count: action.click_count.unwrap_or(1), + }) + } + + async fn poll_successor( + backend: &dyn Backend, + previous: &crate::state::Observation, + condition: Option<&UiCondition>, + preexisting: bool, + ) -> Result<(RootObservation, &'static str, bool), crate::backend::BackendError> { + let timeout = Duration::from_millis( + condition + .and_then(|condition| condition.timeout_ms) + .unwrap_or(DEFAULT_TIMEOUT_MS) + .min(MAX_TIMEOUT_MS), + ); + let deadline = Instant::now() + timeout; + loop { + let (mut observed, root_changed) = observe_with_root_fallback( + backend, + &previous.root, + ObserveRequest { + semantic: true, + capture: CapturePolicy::Never, + }, + )?; + outline::assign_refs_from_previous(&previous.tree, &mut observed.tree); + let status = match condition { + None => Some("not_requested"), + Some(_) if preexisting => Some("preexisting"), + Some(condition) if condition_satisfied(&observed.tree, condition) => { + Some("verified") + } + Some(_) if Instant::now() >= deadline => Some("failed"), + Some(_) => None, + }; + if let Some(status) = status { + return Ok((observed, status, root_changed)); + } + tokio::time::sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; + } + } + + fn observe_with_root_fallback( + backend: &dyn Backend, + root: &RootInfo, + request: ObserveRequest, + ) -> Result<(RootObservation, bool), crate::backend::BackendError> { + match backend.observe(root, request) { + Ok(observed) => Ok((observed, false)), + Err(original_error) => { + let discovered = backend.list_roots(&RootFilters::default())?; + let discovered = roots().lock().unwrap().refresh(discovered); + let Some(successor_root) = discovered.into_iter().next() else { + return Err(original_error); + }; + backend + .observe(&successor_root, request) + .map(|observed| (observed, successor_root.identity() != root.identity())) + } + } + } + + fn condition_satisfied(tree: &UiNode, condition: &UiCondition) -> bool { + let scope = match condition.scope_ref.as_deref() { + Some(ref_id) => tree.find(ref_id), + None => Some(tree), + }; + let Some(scope) = scope else { + return matches!(condition.until, Some(ConditionUntil::Absent)); + }; + let present = if let Some(ref_id) = condition.r#ref.as_deref() { + scope + .find(ref_id) + .is_some_and(|node| node_matches(node, condition)) + } else { + any_node_matches(scope, condition) + }; + match condition.until.unwrap_or(ConditionUntil::Present) { + ConditionUntil::Present => present, + ConditionUntil::Absent => !present, + } + } + + fn any_node_matches(node: &UiNode, condition: &UiCondition) -> bool { + node_matches(node, condition) + || node + .children + .iter() + .any(|child| any_node_matches(child, condition)) + } + + fn node_matches(node: &UiNode, condition: &UiCondition) -> bool { + let text_matches = condition.text.as_deref().is_none_or(|text| { + contains_case_insensitive(&node.title, text) + || contains_case_insensitive(&node.value, text) + || contains_case_insensitive(&node.description, text) + }); + let role_matches = condition.role.as_deref().is_none_or(|role| { + outline::canonical_role(&node.role) == outline::canonical_role(role) + }); + let value_matches = condition + .value + .as_deref() + .is_none_or(|value| contains_case_insensitive(&node.value, value)); + text_matches && role_matches && value_matches + } + + fn contains_case_insensitive(haystack: &str, needle: &str) -> bool { + haystack.to_lowercase().contains(&needle.to_lowercase()) + } + + fn until_name(until: Option) -> &'static str { + match until.unwrap_or(ConditionUntil::Present) { + ConditionUntil::Present => "present", + ConditionUntil::Absent => "absent", + } + } + + fn action_name(action: UiActionKind) -> &'static str { + match action { + UiActionKind::Press => "press", + UiActionKind::Click => "click", + UiActionKind::SetText => "set_text", + UiActionKind::TypeText => "type_text", + UiActionKind::Keypress => "keypress", + UiActionKind::Scroll => "scroll", + UiActionKind::Drag => "drag", + UiActionKind::MoveMouse => "move_mouse", + } + } + + fn count_nodes(node: &UiNode) -> usize { + 1 + node.children.iter().map(count_nodes).sum::() + } + + fn escaped(value: &str) -> String { + value + .replace('\\', "\\\\") + .replace('"', "\\\"") + .replace(['\n', '\r'], " ") + } + + fn bounded_success( + owner_state: Option<&str>, + text: String, + mut extra: Vec, + ) -> CallToolResult { + let text = crate::state::global() + .lock() + .unwrap() + .bound_model_text(owner_state, text); + let mut content = vec![Content::text(text)]; + content.append(&mut extra); + CallToolResult::success(content) + } + + fn backend_error(error: crate::backend::BackendError) -> CallToolResult { + let text = serde_json::to_string(&error).unwrap_or_else(|_| error.to_string()); + CallToolResult::error(vec![Content::text(text)]) + } + + fn tool_error(message: &str) -> CallToolResult { + CallToolResult::error(vec![Content::text(message)]) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn all_tools_are_registered() { + let tools = ComputerUseTools::new(); + let mut names: Vec<_> = tools + .tool_router + .list_all() + .into_iter() + .map(|tool| tool.name.to_string()) + .collect(); + names.sort(); + assert_eq!( + names, + [ + "act_ui", + "expand_ui", + "find_roots", + "inspect_ui", + "observe_ui", + "read_text", + "search_ui", + "wait_for", + ] + ); + } +} diff --git a/crates/core/src/settings.rs b/crates/core/src/settings.rs index c1625c7a..09c96d0b 100644 --- a/crates/core/src/settings.rs +++ b/crates/core/src/settings.rs @@ -491,6 +491,79 @@ impl TitleGenerationSettings { } } +/// When a computer-use observation carries a screenshot alongside the folded +/// accessibility outline. Mirrors `computer_use_mcp::config::ImageMode`; kept in +/// core so settings stay GPUI/backend-free and the app maps one to the other. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ImageMode { + /// Screenshot only when the outline looks too sparse to act on (default). + #[default] + Auto, + /// Always attach a screenshot. + Always, + /// Never attach a screenshot (outline only). + Never, +} + +/// Global configuration for desktop computer-use tools. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ComputerUseSettings { + /// Whether newly spawned provider sessions receive the computer-use MCP server. + #[serde(default)] + pub enabled: bool, + /// When observations include a screenshot. Absent in legacy files → `auto`. + #[serde(default)] + pub image_mode: ImageMode, + /// When false the tools are observe-only (`act_ui` rejects every action). + /// Defaults to TRUE and tolerates an absent field in legacy files. + #[serde(default = "default_true")] + pub allow_input: bool, +} + +impl Default for ComputerUseSettings { + fn default() -> Self { + Self { + enabled: false, + image_mode: ImageMode::default(), + allow_input: true, + } + } +} + +/// Settings for the embedded preview browser (Settings → Browser) and the +/// `tcode_preview` MCP server it backs. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct BrowserSettings { + /// Whether the embedded browser and its preview MCP tools are available. + /// Defaults to TRUE; absent in legacy files → enabled. + #[serde(default = "default_true")] + pub enabled: bool, + /// Initial page opened when the preview panel is shown without a target. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub home_url: Option, + /// Whether the `preview_evaluate` MCP tool may run JavaScript. Defaults to + /// TRUE; absent in legacy files → allowed. + #[serde(default = "default_true")] + pub allow_evaluate: bool, +} + +impl Default for BrowserSettings { + fn default() -> Self { + Self { + enabled: true, + home_url: None, + allow_evaluate: true, + } + } +} + +impl BrowserSettings { + fn is_default(&self) -> bool { + self == &Self::default() + } +} + // `Eq` is intentionally absent: `acp_agents` holds `AcpLaunch`, which the // agent crate derives only `PartialEq` for. #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] @@ -542,6 +615,14 @@ pub struct Settings { /// Built-in orchestration identities and child-model routing table. #[serde(default, skip_serializing_if = "OrchestrateSettings::is_default")] pub orchestrate: OrchestrateSettings, + /// Global desktop computer-use feature settings. Absent in legacy files, + /// where the feature remains disabled by default. + #[serde(default)] + pub computer_use: ComputerUseSettings, + /// Embedded preview browser settings. Skipped when default (like + /// `orchestrate`), so legacy files stay clean and load with the defaults. + #[serde(default, skip_serializing_if = "BrowserSettings::is_default")] + pub browser: BrowserSettings, /// Provider/model used to generate a concise title for new threads. #[serde(default, skip_serializing_if = "TitleGenerationSettings::is_default")] pub title_generation: TitleGenerationSettings, @@ -861,6 +942,70 @@ mod tests { assert_eq!(back.orchestrate, settings.orchestrate); } + #[test] + fn computer_use_defaults_disabled_and_round_trips() { + let legacy: Settings = serde_json::from_str(r#"{"theme_mode":"system"}"#).unwrap(); + assert!(!legacy.computer_use.enabled); + // New fields tolerate an absent block: image mode auto, input allowed. + assert_eq!(legacy.computer_use.image_mode, ImageMode::Auto); + assert!(legacy.computer_use.allow_input); + + // A legacy block that predates image_mode / allow_input still defaults + // input ON (observe-only is opt-in, never the silent legacy behavior). + let partial: Settings = + serde_json::from_str(r#"{"computer_use":{"enabled":true}}"#).unwrap(); + assert!(partial.computer_use.enabled); + assert_eq!(partial.computer_use.image_mode, ImageMode::Auto); + assert!(partial.computer_use.allow_input); + + let settings = Settings { + computer_use: ComputerUseSettings { + enabled: true, + image_mode: ImageMode::Always, + allow_input: false, + }, + ..Settings::default() + }; + let json = serde_json::to_string(&settings).unwrap(); + assert!(json.contains(r#""image_mode":"always""#)); + let back: Settings = serde_json::from_str(&json).unwrap(); + assert!(back.computer_use.enabled); + assert_eq!(back.computer_use.image_mode, ImageMode::Always); + assert!(!back.computer_use.allow_input); + } + + #[test] + fn browser_defaults_enabled_and_round_trips() { + // Legacy files (no `browser` key) get the defaults: enabled, no home + // URL, evaluate allowed. + let legacy: Settings = serde_json::from_str(r#"{"theme_mode":"system"}"#).unwrap(); + assert_eq!(legacy.browser, BrowserSettings::default()); + assert!(legacy.browser.enabled); + assert!(legacy.browser.allow_evaluate); + assert_eq!(legacy.browser.home_url, None); + + // A partial block keeps unspecified fields at their (true) defaults. + let partial: Settings = serde_json::from_str(r#"{"browser":{"enabled":false}}"#).unwrap(); + assert!(!partial.browser.enabled); + assert!(partial.browser.allow_evaluate); + + // Default browser settings are skipped on serialize, like orchestrate. + let json = serde_json::to_string(&Settings::default()).unwrap(); + assert!(!json.contains("\"browser\"")); + + let settings = Settings { + browser: BrowserSettings { + enabled: false, + home_url: Some("https://example.test".into()), + allow_evaluate: false, + }, + ..Settings::default() + }; + let json = serde_json::to_string(&settings).unwrap(); + let back: Settings = serde_json::from_str(&json).unwrap(); + assert_eq!(back.browser, settings.browser); + } + #[test] fn orchestrate_child_approval_defaults_and_round_trips() { let legacy: OrchestrateSettings = serde_json::from_str("{}").unwrap(); diff --git a/crates/runtime/Cargo.toml b/crates/runtime/Cargo.toml index 9af953c5..e3f31e9f 100644 --- a/crates/runtime/Cargo.toml +++ b/crates/runtime/Cargo.toml @@ -6,6 +6,7 @@ edition = "2024" [dependencies] agent = { path = "../agent" } async-channel = "2" +computer-use-mcp = { path = "../computer-use-mcp" } gpui = { git = "https://github.com/zed-industries/zed" } log = "0.4" orchestrate-mcp = { path = "../orchestrate-mcp" } diff --git a/crates/runtime/src/app.rs b/crates/runtime/src/app.rs index b0460b0c..1a09da4c 100644 --- a/crates/runtime/src/app.rs +++ b/crates/runtime/src/app.rs @@ -23,8 +23,8 @@ use tcode_core::provider_models::{ResolvedModel, picker_models, resolve_models}; use tcode_core::provider_status::ProviderSnapshot; use tcode_core::session::{EntryContent, ReviewComment, Timeline, implement_prompt, plan_title}; use tcode_core::settings::{ - ChildApprovalMode, EnvVar, OrchestrateSettings, ProjectSort, ProviderProfile, ProviderSettings, - ResolvedProfile, Settings, provider_label, + ChildApprovalMode, EnvVar, ImageMode, OrchestrateSettings, ProjectSort, ProviderProfile, + ProviderSettings, ResolvedProfile, Settings, provider_label, }; use tcode_services::acp_registry::{ Registry, RegistryAgent, cached, install, load, platform_key, resolve_recipe, uninstall, @@ -698,6 +698,9 @@ pub struct AppState { orchestrate_registrations: HashMap, /// Requests from the orchestrate MCP runtime, pumped on the gpui thread. pub orchestrate_requests: Option>, + /// Process-wide computer-use MCP registration, supplied only to sessions + /// while the global computer-use setting is enabled. + computer_use_registration: Option, callback_last_turn: HashMap, callback_approval_requests: HashSet<(String, String)>, /// Live provider approvals for sessions without an authoritative active @@ -733,6 +736,23 @@ pub struct AppState { /// Per-provider install/auth probe results, driving the Settings → Providers /// card status dot + summary line. Absent until the first probe lands. pub provider_snapshots: HashMap, + /// A restart-continuity marker taken at launch (see `tcode_services::relaunch`). + /// Present only after an app-relaunch triggered by a permission grant; applied + /// once by [`AppState::apply_pending_relaunch`] and then cleared. + pending_relaunch: Option, +} + +/// Map core's persisted computer-use settings onto the live MCP config type +/// (kept separate so `core` stays free of the computer-use backend dependency). +fn computer_use_config(settings: &Settings) -> computer_use_mcp::config::ComputerUseConfig { + computer_use_mcp::config::ComputerUseConfig { + allow_input: settings.computer_use.allow_input, + image_mode: match settings.computer_use.image_mode { + ImageMode::Auto => computer_use_mcp::config::ImageMode::Auto, + ImageMode::Always => computer_use_mcp::config::ImageMode::Always, + ImageMode::Never => computer_use_mcp::config::ImageMode::Never, + }, + } } impl EventEmitter for AppState {} @@ -749,6 +769,12 @@ impl AppState { let projects = file.projects; let settings_store = SettingsStore::new(store.root().clone()); let settings = settings_store.load(); + // Push the loaded computer-use config to the (already-running) MCP layer + // so the tools honor the persisted image-mode / allow-input choices from + // the first call, not just after a settings change. + computer_use_mcp::config::set(computer_use_config(&settings)); + // Consume any restart-continuity marker left by a permission grant. + let pending_relaunch = tcode_services::relaunch::take(store.root()); let settings_collapsed = settings.sidebar_collapsed; let terminal_preferences_path = store.root().join("terminal-ui.json"); let terminal_preferences = std::fs::read(&terminal_preferences_path) @@ -812,6 +838,7 @@ impl AppState { orchestrate_tokens: None, orchestrate_registrations: HashMap::new(), orchestrate_requests: None, + computer_use_registration: None, callback_last_turn: HashMap::new(), callback_approval_requests: HashSet::new(), sessions_awaiting_approval: HashMap::new(), @@ -830,6 +857,7 @@ impl AppState { debug_provider_expanded: None, provider_versions: HashMap::new(), provider_snapshots: HashMap::new(), + pending_relaunch, } } @@ -870,6 +898,14 @@ impl AppState { self.orchestrate_requests = Some(server.requests); } + pub fn attach_computer_use_mcp(&mut self, url: String, token: String) { + self.computer_use_registration = Some(agent::McpRegistration { + name: agent::McpRegistration::SERVER_NAME_COMPUTER_USE.into(), + url, + bearer_token: token, + }); + } + /// Pump orchestrator requests through the runtime on the gpui thread. /// /// Taking the receiver makes repeated calls harmless: exactly one pump can @@ -3262,10 +3298,44 @@ impl AppState { } let language = settings.language.clone(); self.settings = settings; + // Keep the live computer-use MCP config in step with the persisted + // settings on every change (the server outlives any one snapshot). + computer_use_mcp::config::set(computer_use_config(&self.settings)); cx.emit(AppEvent::Effect(RuntimeEffect::ApplyLocale { language })); cx.notify(); } + /// Persist a restart-continuity marker naming the Settings page to reopen and + /// the session that is active now. Written *before* a permission grant or an + /// explicit relaunch, so an externally-initiated quit reopens cleanly. + pub fn write_relaunch_marker(&self, reopen_settings: &str) { + let marker = tcode_services::relaunch::RelaunchMarker { + reopen_settings: reopen_settings.to_string(), + active_session: self.active_session_id().map(str::to_string), + }; + if let Err(err) = tcode_services::relaunch::write(self.store.root(), &marker) { + log::warn!("failed to write relaunch marker: {err}"); + } + } + + /// Apply a marker taken at launch: reopen the recorded session and open + /// Settings on the recorded page. The page reruns a permission recheck as it + /// mounts, so the user immediately sees the post-restart status. No-op when + /// there is no marker (the normal launch path). + pub fn apply_pending_relaunch(&mut self, cx: &mut Context) { + let Some(marker) = self.pending_relaunch.take() else { + return; + }; + if let Some(id) = marker.active_session.as_deref() + && self.sessions.iter().any(|meta| meta.id == id) + { + self.select_session(id, cx); + } + self.debug_settings_section = Some(marker.reopen_settings); + self.route = Route::Settings; + cx.notify(); + } + // -- archive / delete / rename / unread (Group A) ----------------------- /// Archive a thread (reversible; it vanishes from the sidebar). Blocked while @@ -5198,6 +5268,7 @@ impl AppState { let launch_env = self.session_launch_env(&meta); let preview_registration = self.preview_registration_for(&meta); let orchestrate_registration = self.orchestrate_registration_for(&meta); + let computer_use_registration = self.computer_use_registration.clone(); let session_id = meta.id.clone(); if let Some(cursor) = &meta.resume_cursor { log::info!( @@ -5216,6 +5287,7 @@ impl AppState { launch_env, preview_registration, orchestrate_registration, + computer_use_registration, ); let result = start_session(meta.provider, opts).await; let _ = this.update(cx, |state, cx| { @@ -5747,7 +5819,7 @@ impl AppState { let title_meta = title_session_meta(&self.settings, fallback_meta.cwd); let settings = self.settings.clone(); let launch_env = self.session_launch_env(&title_meta); - let options = session_options(&title_meta, &settings, launch_env, None, None); + let options = session_options(&title_meta, &settings, launch_env, None, None, None); let source = first_message.to_string(); let attachments = attachments.to_vec(); @@ -6348,6 +6420,7 @@ fn session_options( launch_env: LaunchEnv, mcp_server: Option, orchestrate_server: Option, + computer_use_server: Option, ) -> SessionOptions { // A session's binary / launch-args come from its selected profile (built-in // or user-created), so a third-party profile can point at its own CLI while @@ -6379,6 +6452,11 @@ fn session_options( } else { None }, + computer_use_server: if settings.computer_use.enabled { + computer_use_server + } else { + None + }, launch_env, // Claude's "Launch arguments"; an ACP agent carries its own from the // installed-agent card (Codex has no such field). @@ -7684,8 +7762,10 @@ mod tests { settings.provider_mut(ProviderKind::ClaudeCode).binary_path = Some(PathBuf::from("/custom/claude")); - let codex_options = session_options(&codex, &settings, LaunchEnv::default(), None, None); - let claude_options = session_options(&claude, &settings, LaunchEnv::default(), None, None); + let codex_options = + session_options(&codex, &settings, LaunchEnv::default(), None, None, None); + let claude_options = + session_options(&claude, &settings, LaunchEnv::default(), None, None, None); assert_eq!( codex_options.binary_path, @@ -7714,7 +7794,7 @@ mod tests { home: settings.provider(ProviderKind::ClaudeCode).effective_home(), }; let meta = SessionMeta::new(ProviderKind::ClaudeCode, PathBuf::from("/x"), None); - let opts = session_options(&meta, &settings, launch_env, None, None); + let opts = session_options(&meta, &settings, launch_env, None, None, None); assert_eq!(opts.extra_args, vec!["--chrome", "--verbose"]); assert_eq!( opts.launch_env.pairs(ProviderKind::ClaudeCode), @@ -7733,7 +7813,7 @@ mod tests { home: settings.provider(ProviderKind::Codex).effective_home(), }; let meta = SessionMeta::new(ProviderKind::Codex, PathBuf::from("/x"), None); - let opts = session_options(&meta, &settings, launch_env, None, None); + let opts = session_options(&meta, &settings, launch_env, None, None, None); assert!(opts.extra_args.is_empty()); assert_eq!( opts.launch_env.pairs(ProviderKind::Codex), @@ -7874,7 +7954,7 @@ mod tests { .iter() .any(|(k, v)| k == "ANTHROPIC_BASE_URL" && v == "https://api.kimi.com/coding/") ); - let opts = session_options(&meta, &state.settings, launch_env, None, None); + let opts = session_options(&meta, &state.settings, launch_env, None, None, None); assert_eq!(opts.binary_path, Some(PathBuf::from("/opt/kimi/claude"))); let _ = std::fs::remove_dir_all(root); @@ -7889,7 +7969,14 @@ mod tests { url: "http://127.0.0.1:7/mcp".into(), bearer_token: "tok".into(), }; - let opts = session_options(&meta, &settings, LaunchEnv::default(), Some(reg), None); + let opts = session_options( + &meta, + &settings, + LaunchEnv::default(), + Some(reg), + None, + None, + ); let mcp = opts.mcp_server.expect("registration threaded through"); assert_eq!(mcp.url, "http://127.0.0.1:7/mcp"); assert_eq!(mcp.bearer_token, "tok"); @@ -7910,6 +7997,7 @@ mod tests { LaunchEnv::default(), None, Some(registration.clone()), + None, ); assert!(normal.mcp_server.is_none()); assert!(normal.orchestrate_server.is_none()); @@ -7921,6 +8009,7 @@ mod tests { LaunchEnv::default(), None, Some(registration), + None, ); assert_eq!( enabled.orchestrate_server.unwrap().name, @@ -7928,6 +8017,41 @@ mod tests { ); } + #[test] + fn session_options_gates_computer_use_registration_on_global_setting() { + let mut settings = Settings::default(); + let meta = SessionMeta::new(ProviderKind::Codex, PathBuf::from("/x"), None); + let registration = agent::McpRegistration { + name: agent::McpRegistration::SERVER_NAME_COMPUTER_USE.into(), + url: "http://127.0.0.1:9/mcp".into(), + bearer_token: "computer-token".into(), + }; + + let disabled = session_options( + &meta, + &settings, + LaunchEnv::default(), + None, + None, + Some(registration.clone()), + ); + assert!(disabled.computer_use_server.is_none()); + + settings.computer_use.enabled = true; + let enabled = session_options( + &meta, + &settings, + LaunchEnv::default(), + None, + None, + Some(registration), + ); + assert_eq!( + enabled.computer_use_server.unwrap().name, + agent::McpRegistration::SERVER_NAME_COMPUTER_USE + ); + } + #[test] fn child_meta_links_parent_project_and_maps_effort() { let mut parent = SessionMeta::new(ProviderKind::ClaudeCode, PathBuf::from("/p"), None); diff --git a/crates/runtime/src/blocking.rs b/crates/runtime/src/blocking.rs index 4228a81a..78eea3e9 100644 --- a/crates/runtime/src/blocking.rs +++ b/crates/runtime/src/blocking.rs @@ -64,6 +64,7 @@ mod tests { let expected = [ "agent", "app", + "computer-use-mcp", "core", "i18n", "orchestrate-mcp", diff --git a/crates/runtime/src/ui_facade.rs b/crates/runtime/src/ui_facade.rs index 33235738..972bffd2 100644 --- a/crates/runtime/src/ui_facade.rs +++ b/crates/runtime/src/ui_facade.rs @@ -15,11 +15,6 @@ pub fn open_in_zed(cwd: &Path) -> io::Result<()> { tcode_services::desktop::open_in_zed(cwd) } -#[cfg(target_os = "macos")] -pub fn capture_screen_region(region: &str) -> Result, String> { - tcode_services::desktop::capture_screen_region(region) -} - pub fn read_file_bytes(path: &Path) -> io::Result> { tcode_services::user_files::read_bytes(path) } diff --git a/crates/services/src/desktop.rs b/crates/services/src/desktop.rs index e3383ca9..07c2a12f 100644 --- a/crates/services/src/desktop.rs +++ b/crates/services/src/desktop.rs @@ -6,24 +6,3 @@ pub fn open_in_zed(cwd: &Path) -> io::Result<()> { drop(child); Ok(()) } - -#[cfg(target_os = "macos")] -pub fn capture_screen_region(region: &str) -> Result, String> { - let path = std::env::temp_dir().join(format!("tcode-preview-{}.png", uuid::Uuid::new_v4())); - let result = crate::process::command("screencapture") - .arg("-x") - .arg("-R") - .arg(region) - .arg(&path) - .status() - .map_err(|err| format!("failed to run screencapture: {err}")) - .and_then(|status| { - if status.success() { - std::fs::read(&path).map_err(|err| format!("failed to read screenshot: {err}")) - } else { - Err("screencapture failed".to_string()) - } - }); - let _ = std::fs::remove_file(path); - result -} diff --git a/crates/services/src/lib.rs b/crates/services/src/lib.rs index 784b7b69..cea7eb91 100644 --- a/crates/services/src/lib.rs +++ b/crates/services/src/lib.rs @@ -7,6 +7,7 @@ pub mod import; pub mod process; pub mod provider_auth; pub mod provider_probe; +pub mod relaunch; pub mod settings; pub mod shell_env; pub mod store; diff --git a/crates/services/src/process.rs b/crates/services/src/process.rs index b5382567..8b34e7e9 100644 --- a/crates/services/src/process.rs +++ b/crates/services/src/process.rs @@ -129,6 +129,7 @@ mod tests { let expected = [ "agent", "app", + "computer-use-mcp", "core", "i18n", "orchestrate-mcp", diff --git a/crates/services/src/relaunch.rs b/crates/services/src/relaunch.rs new file mode 100644 index 00000000..22b9208b --- /dev/null +++ b/crates/services/src/relaunch.rs @@ -0,0 +1,87 @@ +//! Restart-continuity marker (`docs/computer-use.md` §Restart continuity). +//! +//! macOS applies some TCC grants (notably Screen Recording) only after the app +//! restarts, and may quit tcode from its own "Quit & Reopen" dialog. Before any +//! permission flow, the app drops a small `relaunch.json` marker into the data +//! dir recording which Settings page to reopen and which session was active. +//! On the next launch the marker is *taken* (read + deleted) so the app can +//! reopen the session, reopen Settings on the recorded page, and recheck. + +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +/// A pending restart's continuity state. Written before a grant/relaunch and +/// consumed exactly once at the next startup. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RelaunchMarker { + /// Which Settings page to reopen: `"computer_use"` or `"browser"`. + pub reopen_settings: String, + /// The session that was active when the marker was written, if any. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub active_session: Option, +} + +/// The marker file inside the data dir. +fn marker_path(data_dir: &Path) -> PathBuf { + data_dir.join("relaunch.json") +} + +/// Persist the marker, overwriting any previous one. +pub fn write(data_dir: &Path, marker: &RelaunchMarker) -> std::io::Result<()> { + let data = serde_json::to_vec_pretty(marker) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + std::fs::write(marker_path(data_dir), data) +} + +/// Read the marker and delete it (consume-once). Returns `None` when absent or +/// unparsable; the file is removed either way so a corrupt marker can't wedge +/// every future launch into a relaunch loop. +pub fn take(data_dir: &Path) -> Option { + let path = marker_path(data_dir); + let bytes = std::fs::read(&path).ok()?; + let _ = std::fs::remove_file(&path); + serde_json::from_slice(&bytes).ok() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn write_then_take_consumes_the_marker() { + let root = + std::env::temp_dir().join(format!("tcode-relaunch-test-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&root).unwrap(); + + // No marker yet. + assert_eq!(take(&root), None); + + let marker = RelaunchMarker { + reopen_settings: "computer_use".into(), + active_session: Some("sess-42".into()), + }; + write(&root, &marker).unwrap(); + + // Taking returns it exactly once, then the file is gone. + assert_eq!(take(&root), Some(marker)); + assert!(!marker_path(&root).exists()); + assert_eq!(take(&root), None); + + let _ = std::fs::remove_dir_all(root); + } + + #[test] + fn take_removes_a_corrupt_marker() { + let root = + std::env::temp_dir().join(format!("tcode-relaunch-corrupt-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&root).unwrap(); + std::fs::write(marker_path(&root), b"not json").unwrap(); + + assert_eq!(take(&root), None); + // A corrupt marker is deleted so it can't wedge future launches. + assert!(!marker_path(&root).exists()); + + let _ = std::fs::remove_dir_all(root); + } +} diff --git a/crates/services/src/settings.rs b/crates/services/src/settings.rs index fd3a93ac..f6235bed 100644 --- a/crates/services/src/settings.rs +++ b/crates/services/src/settings.rs @@ -197,6 +197,8 @@ mod tests { auto_open_task_panel: true, provider_update_checks_disabled: true, orchestrate: Default::default(), + computer_use: Default::default(), + browser: Default::default(), title_generation: Default::default(), collapsed_projects: vec!["proj-a".into(), "proj-b".into()], favorite_models: vec!["opus".into()], diff --git a/crates/ui/Cargo.toml b/crates/ui/Cargo.toml index 2eaa6b39..68dc72fa 100644 --- a/crates/ui/Cargo.toml +++ b/crates/ui/Cargo.toml @@ -10,6 +10,7 @@ tcode-i18n = { path = "../i18n" } tcode-runtime = { path = "../runtime" } term = { path = "../term" } preview-mcp = { path = "../preview-mcp" } +computer-use-mcp = { path = "../computer-use-mcp" } gpui = { git = "https://github.com/zed-industries/zed" } gpui-component = { git = "https://github.com/longbridge/gpui-component" } gpui-component-assets = { git = "https://github.com/longbridge/gpui-component" } @@ -28,6 +29,15 @@ gpui-wry = { git = "https://github.com/longbridge/gpui-component" } wry = { version = "0.53.3", package = "lb-wry" } raw-window-handle = { version = "0.6", features = ["std"] } +[target.'cfg(target_os = "macos")'.dependencies] +block2 = "0.6.2" +objc2-app-kit = { version = "0.3.2", features = ["NSGraphicsContext", "NSImage", "NSImageRep", "objc2-core-graphics"] } +objc2-core-foundation = { version = "0.3.2", features = ["CFData", "CFString"] } +objc2-core-graphics = { version = "0.3.2", features = ["CGImage"] } +objc2-foundation = { version = "0.3.2", features = ["NSError"] } +objc2-image-io = { version = "0.3.2", features = ["CGImageDestination"] } +objc2-web-kit = { version = "0.3.2", features = ["WKSnapshotConfiguration", "WKWebView"] } + [dev-dependencies] criterion = "0.7" gpui = { git = "https://github.com/zed-industries/zed", features = ["test-support"] } diff --git a/crates/ui/src/preview_panel.rs b/crates/ui/src/preview_panel.rs index 91d8e180..119df878 100644 --- a/crates/ui/src/preview_panel.rs +++ b/crates/ui/src/preview_panel.rs @@ -397,6 +397,20 @@ mod native { let key = self.routed_key(&session_id, cx); log::info!("preview: handling op {op:?} for session {session_id}"); + // Gate on the Browser settings: a disabled browser rejects every op; + // `allow_evaluate` gates only `preview_evaluate`. + let browser = self.app_state.read(cx).settings.browser.clone(); + if !browser.enabled { + let _ = reply.try_send(Err(tcode_i18n::tr!("browser.disabled_error").into_owned())); + return; + } + if matches!(&op, PreviewOp::Evaluate { .. }) && !browser.allow_evaluate { + let _ = reply.try_send(Err( + tcode_i18n::tr!("browser.evaluate_disabled_error").into_owned() + )); + return; + } + match op { PreviewOp::Open { url } => { self.app_state.update(cx, |state, cx| { @@ -404,6 +418,14 @@ mod native { }); if let Some(url) = url.as_deref() { self.navigate(&key, url, window, cx); + } else if let Some(home) = browser + .home_url + .as_deref() + .map(str::trim) + .filter(|home| !home.is_empty()) + { + // No explicit target: fall back to the configured home URL. + self.navigate(&key, home, window, cx); } else { self.ensure_webview(&key, window, cx); self.sync_visibility(cx); @@ -507,24 +529,25 @@ mod native { } } - /// Capture the WebView's on-screen region with `screencapture` (wry exposes - /// no capture API) and answer with a base64 PNG. Best-effort geometry: the - /// region is the window origin plus the WebView's laid-out bounds. + /// Snapshot the native WKWebView in-process and answer with a base64 PNG. /// - /// macOS only. Windows has no comparable CLI and Wayland forbids screen - /// capture outright, so elsewhere the tool reports a normal MCP error - /// rather than pretending. + /// macOS only. Elsewhere the tool reports a normal MCP error rather than + /// pretending to have a portable native-webview snapshot implementation. #[cfg(target_os = "macos")] fn screenshot( &mut self, session_id: &str, key: &str, reply: ReplyTx, - window: &mut Window, + _window: &mut Window, cx: &mut Context, ) { - use base64::Engine as _; + use block2::RcBlock; use gpui::px; + use objc2_app_kit::NSImage; + use objc2_foundation::NSError; + use objc2_web_kit::WKWebView; + use wry::WebViewExtMacOS as _; let visible = { let state = self.app_state.read(cx); @@ -558,21 +581,28 @@ mod native { let _ = reply.try_send(Err("preview browser has no visible area".into())); return; } - let window_origin = window.bounds().origin; - let region = super::screen_region(window_origin, wv_bounds); - - match tcode_runtime::ui_facade::capture_screen_region(®ion) { - Ok(bytes) => { - let data_base64 = base64::engine::general_purpose::STANDARD.encode(&bytes); - let _ = reply.try_send(Ok(PreviewReply::Image { - mime: "image/png".into(), - data_base64, - })); - } - Err(err) => { - let _ = reply.try_send(Err(err)); - } + let native = view.read(cx).raw().webview(); + let webview: &WKWebView = &native; + let callback_reply = reply.clone(); + let handler = RcBlock::new(move |image: *mut NSImage, error: *mut NSError| { + let result = if let Some(image) = unsafe { image.as_ref() } { + super::snapshot_reply(image) + } else if !error.is_null() { + Err("WKWebView snapshot failed".into()) + } else { + Err("WKWebView snapshot returned no image".into()) + }; + let _ = callback_reply.try_send(result); + }); + unsafe { + webview.takeSnapshotWithConfiguration_completionHandler(None, &handler); } + + cx.spawn(async move |_, cx| { + cx.background_executor().timer(Duration::from_secs(5)).await; + let _ = reply.try_send(Err("WKWebView snapshot timed out after 5 seconds".into())); + }) + .detach(); } /// See the macOS implementation: screen capture has no portable @@ -592,6 +622,18 @@ mod native { impl Render for PreviewPanel { fn render(&mut self, window: &mut Window, cx: &mut Context) -> impl IntoElement { + // When the embedded browser is turned off in Settings → Browser, hide + // the chrome and webview entirely and show a quiet placeholder. + if !self.app_state.read(cx).settings.browser.enabled { + return v_flex() + .size_full() + .items_center() + .justify_center() + .px_8() + .text_center() + .text_color(cx.theme().muted_foreground) + .child(tcode_i18n::tr!("browser.disabled_panel")); + } let active = self.active_key(cx); // Honor a queued `--open-preview ` navigation once a session exists. @@ -798,25 +840,36 @@ mod placeholder { } } -/// The error `preview_screenshot` reports where screen capture has no reliable -/// implementation (Windows has no `screencapture` equivalent; Wayland forbids it -/// outright). Linux has no webview at all, so it never gets this far. +/// The error `preview_screenshot` reports where native-webview snapshots have no +/// implementation. Linux has no webview at all, so it never gets this far. #[cfg(not(target_os = "linux"))] #[cfg_attr(target_os = "macos", allow(dead_code))] const SCREENSHOT_UNSUPPORTED: &str = "preview_screenshot is only supported on macOS"; -/// Compute a `screencapture -R x,y,w,h` region string from the window origin and -/// the WebView's window-relative bounds. macOS-only, like its one caller. #[cfg(target_os = "macos")] -fn screen_region( - window_origin: gpui::Point, - wv: gpui::Bounds, -) -> String { - let x = f32::from(window_origin.x + wv.origin.x).round() as i32; - let y = f32::from(window_origin.y + wv.origin.y).round() as i32; - let w = f32::from(wv.size.width).round() as i32; - let h = f32::from(wv.size.height).round() as i32; - format!("{x},{y},{w},{h}") +fn snapshot_reply(image: &objc2_app_kit::NSImage) -> Result { + use base64::Engine as _; + use objc2_core_foundation::{CFMutableData, CFString}; + use objc2_image_io::CGImageDestination; + + let cg_image = + unsafe { image.CGImageForProposedRect_context_hints(std::ptr::null_mut(), None, None) } + .ok_or_else(|| "failed to obtain CGImage from WKWebView snapshot".to_string())?; + let data = CFMutableData::new(None, 0) + .ok_or_else(|| "failed to allocate PNG destination data".to_string())?; + let png_type = CFString::from_static_str("public.png"); + let destination = unsafe { CGImageDestination::with_data(&data, &png_type, 1, None) } + .ok_or_else(|| "failed to create PNG image destination".to_string())?; + unsafe { + destination.add_image(&cg_image, None); + if !destination.finalize() { + return Err("failed to finalize WKWebView snapshot PNG".into()); + } + } + Ok(PreviewReply::Image { + mime: "image/png".into(), + data_base64: base64::engine::general_purpose::STANDARD.encode(data.to_vec()), + }) } /// What an automation tool answers when the platform webview cannot be created @@ -904,23 +957,6 @@ mod tests { assert_eq!(visible_preview_key(None, Route::Chat, false, true), None); } - /// The capture region is the window origin plus the WebView's own bounds. - /// macOS-only, like the `screencapture` shell-out it feeds. - #[cfg(target_os = "macos")] - #[test] - fn screen_region_is_the_window_origin_plus_the_webview_bounds() { - assert_eq!( - screen_region( - gpui::point(gpui::px(10.), gpui::px(20.)), - gpui::Bounds { - origin: gpui::point(gpui::px(5.), gpui::px(5.)), - size: gpui::size(gpui::px(100.), gpui::px(50.)), - } - ), - "15,25,100,50" - ); - } - /// Off macOS (but where a webview exists — i.e. Windows) `preview_screenshot` /// surfaces a plain tool error instead of a broken capture. On Linux there is /// no webview at all and the whole panel is a placeholder. diff --git a/crates/ui/src/settings.rs b/crates/ui/src/settings.rs index 3b922651..b0a80107 100644 --- a/crates/ui/src/settings.rs +++ b/crates/ui/src/settings.rs @@ -36,9 +36,9 @@ pub fn apply_locale(override_locale: Option<&str>) { } pub use tcode_core::settings::{ - ChildApprovalMode, EnvVar, OrchestrateChildModel, OrchestrateSettings, OrchestratorIdentity, - ProjectSort, ProviderSettings, Settings, ThemeMode, TitleGenerationSettings, provider_key, - provider_label, + BrowserSettings, ChildApprovalMode, ComputerUseSettings, EnvVar, ImageMode, + OrchestrateChildModel, OrchestrateSettings, OrchestratorIdentity, ProjectSort, + ProviderSettings, Settings, ThemeMode, TitleGenerationSettings, provider_key, provider_label, }; /// The six accent presets offered by the provider card (T3 §2). pub const ACCENT_PRESETS: [&str; 6] = [ diff --git a/crates/ui/src/settings_page.rs b/crates/ui/src/settings_page.rs index 16e8e08b..eae51169 100644 --- a/crates/ui/src/settings_page.rs +++ b/crates/ui/src/settings_page.rs @@ -16,18 +16,25 @@ use gpui_component::{ ThemeMode as ComponentThemeMode, WindowExt as _, button::{Button, ButtonVariant, ButtonVariants as _}, dialog::DialogButtonProps, + input::{Input, InputEvent, InputState}, popover::Popover, switch::Switch, v_flex, }; +use computer_use_mcp::permissions::{ + self, PermissionKind, PermissionStatus, open_settings_pane, relaunch_app, request, +}; use tcode_runtime::app::AppState; use crate::acp_panel::{AcpAgentCard, AcpPanel}; use crate::orchestrate_settings::OrchestrateSettingsPanel; use crate::provider_card::ProviderCard; use crate::provider_model_picker::ProviderModelPicker; -use crate::settings::{LANGUAGE_ENGLISH, LANGUAGE_SIMPLIFIED_CHINESE, Settings, ThemeMode}; +use crate::settings::{ + ImageMode, LANGUAGE_ENGLISH, LANGUAGE_SIMPLIFIED_CHINESE, Settings, ThemeMode, +}; +use crate::shell::Quit; use crate::time::now_secs; use crate::window_drag_area; @@ -46,6 +53,8 @@ const CONTENT_MAX_WIDTH: f32 = 720.; enum Section { General, Providers, + Browser, + ComputerUse, Orchestrate, Archived, } @@ -75,6 +84,14 @@ pub struct SettingsPage { acp_cards: Vec<(String, Entity)>, debug_acp_dialog_pending: bool, section: Section, + /// Editable "Home URL" for the Browser page; committed on change. + home_url_input: Entity, + /// Last-known TCC permission snapshot, refreshed when Computer Use becomes + /// visible and on every explicit Recheck / Grant. + perm_status: PermissionStatus, + /// Whether a Screen Recording grant looks pending-restart (a fresh grant + /// only takes effect after tcode relaunches). Drives the restart banner. + sr_restart_hint: bool, _subscriptions: Vec, } @@ -118,9 +135,12 @@ impl SettingsPage { }), ]; - // Screenshot-only: `--debug-settings-section` opens a specific section. + // Screenshot-only / restart-continuity: `--debug-settings-section` (also + // reused by the relaunch marker) opens a specific section. let section = match app_state.read(cx).debug_settings_section.as_deref() { Some("providers") => Section::Providers, + Some("browser") => Section::Browser, + Some("computer_use") => Section::ComputerUse, Some("orchestrate") => Section::Orchestrate, Some("archived") => Section::Archived, _ => Section::General, @@ -129,6 +149,22 @@ impl SettingsPage { let orchestrate_panel = cx.new(|cx| OrchestrateSettingsPanel::new(app_state.clone(), window, cx)); let debug_acp_dialog_pending = app_state.read(cx).debug_acp_dialog; + let home_url_value = app_state + .read(cx) + .settings + .browser + .home_url + .clone() + .unwrap_or_default(); + let home_url_input = cx.new(|cx| { + InputState::new(window, cx) + .placeholder(tcode_i18n::tr!("browser.home_url.placeholder")) + .default_value(home_url_value) + }); + // Refresh the TCC snapshot once as the page mounts. When the page is + // opened by a post-grant relaunch this is the "automatic recheck" that + // surfaces the new status immediately. + let perm_status = permissions::check(); let mut page = Self { app_state, provider_cards: Vec::new(), @@ -138,13 +174,29 @@ impl SettingsPage { acp_cards: Vec::new(), debug_acp_dialog_pending, section, + home_url_input: home_url_input.clone(), + perm_status, + sr_restart_hint: false, _subscriptions: subscriptions, }; + page._subscriptions + .push(cx.subscribe(&home_url_input, |this, _, event, cx| { + if matches!(event, InputEvent::Change) { + this.commit_home_url(cx); + } + })); page.build_provider_cards(window, cx); page.sync_acp_cards(window, cx); page } + /// Persist the Browser "Home URL" field (empty → `None`). + fn commit_home_url(&self, cx: &mut Context) { + let value = self.home_url_input.read(cx).value().trim().to_string(); + let home_url = (!value.is_empty()).then_some(value); + self.update_settings(move |settings| settings.browser.home_url = home_url, cx); + } + /// (Re)build the provider cards from current settings — also used after /// "Restore defaults", which invalidates every card's inputs. fn build_provider_cards(&mut self, window: &mut Window, cx: &mut Context) { @@ -259,6 +311,11 @@ impl SettingsPage { ) .on_click(cx.listener(move |this, _, _, cx| { this.section = section; + // Refresh the TCC snapshot each time Computer Use becomes + // visible (cheap native calls, event-driven). + if section == Section::ComputerUse { + this.perm_status = permissions::check(); + } cx.notify(); })) .into_any_element() @@ -312,6 +369,22 @@ impl SettingsPage { Section::Providers, cx, )) + .child(nav_item( + self, + "settings-nav-browser", + IconName::Globe, + tcode_i18n::tr!("settings.browser").into_owned().into(), + Section::Browser, + cx, + )) + .child(nav_item( + self, + "settings-nav-computer-use", + IconName::LayoutDashboard, + tcode_i18n::tr!("settings.computer_use").into_owned().into(), + Section::ComputerUse, + cx, + )) .child(nav_item( self, "settings-nav-orchestrate", @@ -409,6 +482,17 @@ impl SettingsPage { let app_state = page.app_state.clone(); page.orchestrate_panel = cx.new(|cx| OrchestrateSettingsPanel::new(app_state, window, cx)); + // The Home URL input now holds a stale override. + let home_url = page + .app_state + .read(cx) + .settings + .browser + .home_url + .clone() + .unwrap_or_default(); + page.home_url_input + .update(cx, |input, cx| input.set_value(home_url, window, cx)); }); apply_theme(ThemeMode::System, window, cx); true @@ -420,6 +504,8 @@ impl SettingsPage { let column = match self.section { Section::General => self.render_general(cx), Section::Providers => self.render_providers(window, cx), + Section::Browser => self.render_browser(cx), + Section::ComputerUse => self.render_computer_use(cx), Section::Orchestrate => v_flex().child(self.orchestrate_panel.clone()), Section::Archived => self.render_archived(cx), }; @@ -721,6 +807,360 @@ impl SettingsPage { }); } + // -- Computer Use & Browser pages -------------------------------------- + + fn render_computer_use(&self, cx: &mut Context) -> gpui::Div { + let settings = self.app_state.read(cx).settings.clone(); + let rows = vec![ + self.toggle_row( + "cu-enabled", + tcode_i18n::tr!("computer_use.enable.title"), + tcode_i18n::tr!("computer_use.enable.description"), + settings.computer_use.enabled, + cx, + |s, checked| s.computer_use.enabled = checked, + ), + self.image_mode_row(settings.computer_use.image_mode, cx), + self.toggle_row( + "cu-allow-input", + tcode_i18n::tr!("computer_use.allow_input.title"), + tcode_i18n::tr!("computer_use.allow_input.description"), + settings.computer_use.allow_input, + cx, + |s, checked| s.computer_use.allow_input = checked, + ), + ]; + v_flex() + .gap(px(24.)) + .child( + v_flex() + .child(self.section_label(tcode_i18n::tr!("computer_use.section"), cx)) + .child(self.grouped(rows, cx)), + ) + .child(self.permissions_group( + &[ + PermissionKind::Accessibility, + PermissionKind::ScreenRecording, + ], + cx, + )) + } + + fn render_browser(&self, cx: &mut Context) -> gpui::Div { + let settings = self.app_state.read(cx).settings.clone(); + let rows = vec![ + self.toggle_row( + "browser-enabled", + tcode_i18n::tr!("browser.enable.title"), + tcode_i18n::tr!("browser.enable.description"), + settings.browser.enabled, + cx, + |s, checked| s.browser.enabled = checked, + ), + self.home_url_row(cx), + self.toggle_row( + "browser-allow-eval", + tcode_i18n::tr!("browser.allow_evaluate.title"), + tcode_i18n::tr!("browser.allow_evaluate.description"), + settings.browser.allow_evaluate, + cx, + |s, checked| s.browser.allow_evaluate = checked, + ), + ]; + v_flex().gap(px(24.)).child( + v_flex() + .child(self.section_label(tcode_i18n::tr!("browser.section"), cx)) + .child(self.grouped(rows, cx)), + ) + } + + fn home_url_row(&self, cx: &mut Context) -> AnyElement { + self.row_frame(cx) + .child(self.row_labels( + tcode_i18n::tr!("browser.home_url.title"), + tcode_i18n::tr!("browser.home_url.description"), + cx, + )) + .child( + div().w(px(240.)).child( + Input::new(&self.home_url_input) + .small() + .rounded(crate::material::radius_input()), + ), + ) + .into_any_element() + } + + fn image_mode_row(&self, mode: ImageMode, cx: &mut Context) -> AnyElement { + let label = match mode { + ImageMode::Auto => tcode_i18n::tr!("computer_use.image_mode.auto"), + ImageMode::Always => tcode_i18n::tr!("computer_use.image_mode.always"), + ImageMode::Never => tcode_i18n::tr!("computer_use.image_mode.never"), + }; + let trigger = self.dropdown_trigger("cu-image-mode-dropdown", label, cx); + let this = cx.entity(); + let dropdown = Popover::new("cu-image-mode-popover") + .trigger(trigger) + .content(move |_, _, cx| { + let this = this.clone(); + let option = |m: ImageMode, + label_key: &'static str, + desc_key: &'static str, + this: &Entity, + cx: &mut Context| + -> AnyElement { + let this = this.clone(); + let popover = cx.entity(); + gpui_component::h_flex() + .id(label_key) + .w_full() + .px_2() + .py_1p5() + .gap_2() + .items_start() + .rounded(crate::material::radius_button()) + .cursor_pointer() + .hover(|s| s.bg(cx.theme().accent)) + .child( + v_flex() + .flex_1() + .gap_0p5() + .child(div().text_size(px(13.)).child(tcode_i18n::tr!(label_key))) + .child( + div() + .text_size(px(11.)) + .text_color(cx.theme().muted_foreground) + .child(tcode_i18n::tr!(desc_key)), + ), + ) + .when(m == mode, |d| d.child(Icon::new(IconName::Check).xsmall())) + .on_click(move |_, window, cx| { + this.update(cx, |page, cx| { + page.update_settings(|s| s.computer_use.image_mode = m, cx); + }); + popover.update(cx, |st, cx| st.dismiss(window, cx)); + }) + .into_any_element() + }; + crate::material::overlay_contour( + v_flex() + .p_1() + .min_w(px(260.)) + .gap_0p5() + .child(option( + ImageMode::Auto, + "computer_use.image_mode.auto", + "computer_use.image_mode.auto_desc", + &this, + cx, + )) + .child(option( + ImageMode::Always, + "computer_use.image_mode.always", + "computer_use.image_mode.always_desc", + &this, + cx, + )) + .child(option( + ImageMode::Never, + "computer_use.image_mode.never", + "computer_use.image_mode.never_desc", + &this, + cx, + )), + cx, + ) + .rounded(crate::material::radius_overlay()) + }); + self.row_frame(cx) + .child(self.row_labels( + tcode_i18n::tr!("computer_use.image_mode.title"), + tcode_i18n::tr!("computer_use.image_mode.description"), + cx, + )) + .child(dropdown) + .into_any_element() + } + + /// The Computer Use "System permissions" group. Non-macOS platforms have + /// no TCC, so it shows a quiet note instead. + fn permissions_group(&self, kinds: &[PermissionKind], cx: &mut Context) -> AnyElement { + let col = v_flex() + .child(self.section_label(tcode_i18n::tr!("computer_use.permissions_section"), cx)); + if !cfg!(target_os = "macos") { + return col + .child( + self.group(cx).child( + div() + .w_full() + .px_3() + .py_3() + .text_size(px(13.)) + .text_color(cx.theme().muted_foreground) + .child(tcode_i18n::tr!("permissions.unsupported")), + ), + ) + .into_any_element(); + } + let rows: Vec = kinds + .iter() + .map(|kind| self.permission_row(*kind, cx)) + .collect(); + let mut stack = v_flex().w_full().gap_2().child(self.grouped(rows, cx)); + // A fresh Screen Recording grant only takes effect after a restart; offer + // an explicit relaunch when we've detected one is pending. + if self.sr_restart_hint && kinds.contains(&PermissionKind::ScreenRecording) { + stack = stack.child(self.restart_banner(cx)); + } + col.child(stack).into_any_element() + } + + fn permission_row(&self, kind: PermissionKind, cx: &mut Context) -> AnyElement { + let granted = self.perm_status.granted(kind); + let (name_key, why_key, grant_id, recheck_id) = match kind { + PermissionKind::Accessibility => ( + "permissions.accessibility.name", + "permissions.accessibility.why", + "perm-grant-accessibility", + "perm-recheck-accessibility", + ), + PermissionKind::ScreenRecording => ( + "permissions.screen_recording.name", + "permissions.screen_recording.why", + "perm-grant-screen-recording", + "perm-recheck-screen-recording", + ), + }; + let mut controls = gpui_component::h_flex() + .flex_none() + .gap_2() + .items_center() + .child(self.status_chip(granted, cx)); + if !granted { + controls = controls + .child( + Button::new(grant_id) + .outline() + .small() + .label(tcode_i18n::tr!("permissions.grant")) + .on_click(cx.listener(move |this, _, _, cx| { + this.grant_permission(kind, cx); + })), + ) + .child( + Button::new(recheck_id) + .ghost() + .small() + .label(tcode_i18n::tr!("permissions.recheck")) + .on_click(cx.listener(|this, _, _, cx| { + this.recheck_permissions(cx); + })), + ); + } + self.row_frame(cx) + .child(self.row_labels(tcode_i18n::tr!(name_key), tcode_i18n::tr!(why_key), cx)) + .child(controls) + .into_any_element() + } + + fn status_chip(&self, granted: bool, cx: &Context) -> AnyElement { + let (bg, fg, label) = if granted { + ( + cx.theme().success.opacity(0.12), + cx.theme().success_foreground, + tcode_i18n::tr!("permissions.granted"), + ) + } else { + ( + cx.theme().warning.opacity(0.15), + cx.theme().warning_foreground, + tcode_i18n::tr!("permissions.missing"), + ) + }; + div() + .flex_none() + .px_2() + .py_0p5() + .rounded_full() + .bg(bg) + .text_size(px(11.)) + .text_color(fg) + .child(label) + .into_any_element() + } + + fn restart_banner(&self, cx: &mut Context) -> AnyElement { + gpui_component::h_flex() + .w_full() + .items_center() + .gap_3() + .rounded(crate::material::radius_card()) + .bg(cx.theme().warning.opacity(0.12)) + .px_3() + .py_2p5() + .child( + Icon::new(IconName::Info) + .small() + .text_color(cx.theme().warning_foreground), + ) + .child( + div() + .flex_1() + .text_size(px(13.)) + .child(tcode_i18n::tr!("permissions.restart_banner")), + ) + .child( + Button::new("perm-relaunch") + .outline() + .small() + .label(tcode_i18n::tr!("permissions.relaunch")) + .on_click(cx.listener(|this, _, window, cx| { + this.relaunch(window, cx); + })), + ) + .into_any_element() + } + + /// Persist the restart-continuity marker, then fire the OS prompt and open + /// the matching System Settings pane. The marker must be written *first*: + /// macOS may quit tcode from its own "Quit & Reopen" dialog. + fn grant_permission(&mut self, kind: PermissionKind, cx: &mut Context) { + self.app_state + .read(cx) + .write_relaunch_marker("computer_use"); + let _ = request(kind); + open_settings_pane(kind); + if kind == PermissionKind::ScreenRecording { + self.sr_restart_hint = true; + } + self.perm_status = permissions::check(); + cx.notify(); + } + + fn recheck_permissions(&mut self, cx: &mut Context) { + let fresh = permissions::check(); + // A Screen Recording grant that flips on still needs a restart to take + // effect for the running process, so surface the relaunch affordance. + if fresh.screen_recording && !self.perm_status.screen_recording { + self.sr_restart_hint = true; + } + self.perm_status = fresh; + cx.notify(); + } + + fn relaunch(&mut self, window: &mut Window, cx: &mut Context) { + self.app_state + .read(cx) + .write_relaunch_marker("computer_use"); + if let Err(err) = relaunch_app() { + log::warn!("failed to relaunch tcode: {err}"); + return; + } + // Quit through the app's existing quit action; the fresh instance + // consumes the marker on launch. + window.dispatch_action(Box::new(Quit), cx); + } + // -- row builders ------------------------------------------------------- /// A group's header: 11px muted caption sitting above its container. diff --git a/docs/computer-use.md b/docs/computer-use.md new file mode 100644 index 00000000..d8fc236a --- /dev/null +++ b/docs/computer-use.md @@ -0,0 +1,105 @@ +# Computer use + +tcode gives every provider (Claude Code, Codex, and all ACP agents that advertise +`mcpCapabilities.http`) a set of desktop computer-use tools, served by the in-process +`tcode_computer_use` MCP server. The design follows +[pi-computer-use](https://github.com/injaneity/pi-computer-use): accessibility-tree-first, +state-scoped observation, transactional actions — not blind pixel clicking. + +## Tool surface + +| Tool | Purpose | +| --- | --- | +| `find_roots` | Ranked list of desktop window roots (`@rN`) with app name, bundle id, pid, title. | +| `observe_ui` | Observe one root (or the frontmost window). Returns a folded accessibility outline with element refs (`@eN`), a `state_id`, and (per image mode) a screenshot. | +| `search_ui` | Ranked text/role search over the full cached outline of a `state_id`. | +| `expand_ui` | Local outline context around one ref, to a given depth. | +| `inspect_ui` | Full attributes, frame, and supported actions for one ref. | +| `act_ui` | Run a transaction of actions (`press`, `click`, `set_text`, `type_text`, `keypress`, `scroll`, `drag`, `move_mouse`) against a `state_id`, optionally with an `expect` postcondition; returns the successor state as a diff or full view. | +| `read_text` | Page through long text owned by a state ref. | +| `wait_for` | Wait for a text/role condition to become present or absent. | + +Core contract, inherited from pi-computer-use: + +- **State-scoped refs.** Every `@e` ref belongs to the `state_id` that produced it. Observations + are immutable and stored in a bounded LRU (default 8). Acting from an evicted or stale state is + rejected with a clear error; the model must observe again. +- **Progressive disclosure.** The first outline is folded; `search_ui` / `expand_ui` / + `inspect_ui` query the full stored tree without touching the live UI. +- **Honest outcomes.** `act_ui` reports `worked` / `didnt` / `unknown` per step, stops at the + first failure (`stopped_at`), and never treats event delivery alone as semantic success when an + `expect` condition was given. +- **Bounded output.** Model-visible text is capped; oversized results return a preview plus a + continuation ref for `read_text`. + +Deliberate v1 deviations from pi-computer-use (documented so later work can close them): +no OCR/`pictureOnly` nodes, no CDP browser roots (browser automation stays on the +`tcode_preview` server and the embedded WebView), no separate helper app (see below), and a +simplified successor-diff heuristic. + +## Architecture + +- `crates/computer-use-mcp` — the whole feature: + - `outline.rs` — platform-neutral UI tree model, folding, search ranking. + - `state.rs` — bounded immutable state store, `state_id` allocation, staleness checks. + - `tools.rs` — rmcp `ToolRouter` (same streamable-HTTP + bearer-token shape as + `preview-mcp` / `orchestrate-mcp`). + - `backend/` — `Backend` trait; `backend/macos/` implements it with the AX C API + (`AXUIElement*`), CGEvent input synthesis, and `screencapture -l ` capture; + other platforms get a stub backend whose tools return a clear "unsupported platform" error. + - `permissions.rs` — TCC checks/requests (see below), public API also consumed by the + settings UI. +- Registration: `SessionOptions.computer_use_server: Option` threaded exactly + like `orchestrate_server` — Claude via `--mcp-config`, Codex via `-c mcp_servers.*`, ACP via + `session/new` `mcpServers` (HTTP-capability-gated). Enabled/disabled per + `Settings.computer_use.enabled`. + +Unlike pi-computer-use, tcode needs **no helper app**: tcode is itself a signed `.app`, so +Accessibility and Screen Recording grants attach directly to tcode. That removes helper +install/signing/attribution handling entirely. + +## macOS permissions + +| Permission | Needed for | Check | Request | +| --- | --- | --- | --- | +| Accessibility | reading AX trees, posting CGEvents | `AXIsProcessTrusted` | `AXIsProcessTrustedWithOptions(prompt)` | +| Screen Recording | computer-use screenshots | `CGPreflightScreenCaptureAccess` | `CGRequestScreenCaptureAccess` | + +Settings gains two pages: + +- **Browser** — enable/disable the embedded preview browser, default home URL, and + allow-JS-evaluate toggle. Its in-process WKWebView snapshot tool needs no TCC permission. +- **Computer Use** — master enable toggle, image mode (`auto` / `always` / `never`), + allow-input-actions toggle (off = observe-only), and one permission row per TCC kind: + live status, a **Grant** button (fires the TCC prompt and opens the matching + `x-apple.systempreferences` pane), and **Recheck**. + +### Restart continuity + +macOS applies some grants (notably Screen Recording) only after the app restarts, and shows its +own "Quit & Reopen" dialog. tcode therefore treats any permission flow as a potential restart: + +1. When the user clicks **Grant**, tcode first writes a small `relaunch.json` marker into the + data dir: `{ reopen_settings: "computer_use", active_session: }`. +2. Session timelines are already continuously persisted (JSONL + resume cursors), so an + externally-initiated quit loses nothing. +3. On startup, a present marker is consumed: the previous active session is reopened, the + Settings window is reopened on the recorded page, and permissions are rechecked + automatically so the user immediately sees the new status. +4. The Computer Use page also offers an explicit **Relaunch tcode** button (shown when a grant + was detected as pending-restart) that writes the same marker and relaunches via + `open -n `. + +## Dev & testing + +- `tcode --cu-permissions` prints the permission status as JSON and exits. +- `tcode --cu-smoke` runs a scripted end-to-end pass without any model: launches TextEdit, + `find_roots` → `observe_ui` → `act_ui` (type text) → verifies the text via a fresh + observation; exit code reflects the verdict. Both flags make VM testing scriptable over SSH. +- Because developing computer use on the dev machine would require the very permissions being + developed (and granting them mid-development churns TCC state), end-to-end testing runs in a + **tart VM**: build on the host, copy the binary in, drive the VM's screen/keyboard over VNC, + grant permissions inside the VM, then run the smoke flags via SSH. +- CI (macOS/Linux/Windows) builds the stub backends and runs the platform-neutral unit tests: + outline folding and search ranking, state-store eviction and staleness, tool schemas, + settings serde round-trips, and MCP registration wiring for all three provider paths. diff --git a/locales/en.yml b/locales/en.yml index c3b46197..4b50e6d1 100644 --- a/locales/en.yml +++ b/locales/en.yml @@ -122,6 +122,8 @@ settings: title: "Settings" general: "General" providers: "Providers" + browser: "Browser" + computer_use: "Computer Use" orchestrate: "Orchestrate" archived: "Archived Threads" archived_section: "ARCHIVED THREADS" @@ -168,6 +170,54 @@ settings: description: "Choose the provider and model used to name new threads. Title requests always use low reasoning effort." model_picker: no_models: "No models available for this provider." +computer_use: + section: "COMPUTER USE" + enable: + title: "Enable computer use" + description: "Give provider sessions the desktop computer-use tools (observe and act on other apps via the accessibility tree)." + image_mode: + title: "Screenshots" + description: "When observations include a screenshot alongside the accessibility outline." + auto: "Auto" + auto_desc: "Only when the outline is too sparse to act on." + always: "Always" + always_desc: "Attach a screenshot to every observation." + never: "Never" + never_desc: "Outline only; never capture the screen." + allow_input: + title: "Allow input actions" + description: "When off, the tools are observe-only: clicks, typing, and other actions are rejected." + permissions_section: "SYSTEM PERMISSIONS" + unsupported: "Computer use is only available on macOS." +browser: + section: "BROWSER" + enable: + title: "Enable embedded browser" + description: "Show the preview panel and expose its browser automation tools to provider sessions." + home_url: + title: "Home URL" + description: "Page opened when the preview panel is shown without a target." + placeholder: "https://example.com" + allow_evaluate: + title: "Allow JavaScript evaluation" + description: "Let the preview_evaluate tool run JavaScript in the embedded browser." + disabled_panel: "The embedded browser is disabled in Settings → Browser." + disabled_error: "The embedded browser is disabled in Settings → Browser." + evaluate_disabled_error: "preview_evaluate is disabled in Settings → Browser." +permissions: + granted: "Granted" + missing: "Not granted" + grant: "Grant" + recheck: "Recheck" + unsupported: "Only available on macOS." + restart_banner: "Screen Recording grants take effect only after tcode restarts." + relaunch: "Relaunch tcode" + accessibility: + name: "Accessibility" + why: "Read the on-screen accessibility tree and post keyboard and mouse events." + screen_recording: + name: "Screen Recording" + why: "Capture window screenshots for computer-use observations." orchestrate: restore_default: "Restore default" child_approval: diff --git a/locales/zh-CN.yml b/locales/zh-CN.yml index 7fcb2e17..24d80dfd 100644 --- a/locales/zh-CN.yml +++ b/locales/zh-CN.yml @@ -122,6 +122,8 @@ settings: title: "设置" general: "通用" providers: "提供商" + browser: "浏览器" + computer_use: "电脑操作" orchestrate: "Orchestrate" archived: "已归档对话" archived_section: "已归档对话" @@ -168,6 +170,54 @@ settings: description: "选择负责为新对话命名的提供方和模型。命名请求始终使用 low 推理强度。" model_picker: no_models: "此提供方暂无可用模型。" +computer_use: + section: "电脑操作" + enable: + title: "启用电脑操作" + description: "为提供方会话提供桌面电脑操作工具（通过辅助功能树观察并操作其他应用）。" + image_mode: + title: "截图" + description: "在观察结果中随辅助功能大纲一并附带截图的时机。" + auto: "自动" + auto_desc: "仅当大纲信息不足以操作时截图。" + always: "始终" + always_desc: "为每次观察都附带截图。" + never: "从不" + never_desc: "仅使用大纲，绝不截屏。" + allow_input: + title: "允许输入操作" + description: "关闭后工具仅用于观察：点击、输入等操作都会被拒绝。" + permissions_section: "系统权限" + unsupported: "电脑操作仅在 macOS 上可用。" +browser: + section: "浏览器" + enable: + title: "启用内嵌浏览器" + description: "显示预览面板，并向提供方会话开放其浏览器自动化工具。" + home_url: + title: "主页地址" + description: "预览面板在没有指定目标时打开的页面。" + placeholder: "https://example.com" + allow_evaluate: + title: "允许执行 JavaScript" + description: "允许 preview_evaluate 工具在内嵌浏览器中运行 JavaScript。" + disabled_panel: "内嵌浏览器已在“设置 → 浏览器”中禁用。" + disabled_error: "内嵌浏览器已在“设置 → 浏览器”中禁用。" + evaluate_disabled_error: "preview_evaluate 已在“设置 → 浏览器”中禁用。" +permissions: + granted: "已授权" + missing: "未授权" + grant: "去授权" + recheck: "重新检查" + unsupported: "仅在 macOS 上可用。" + restart_banner: "屏幕录制授权需重新启动 tcode 后才会生效。" + relaunch: "重新启动 tcode" + accessibility: + name: "辅助功能" + why: "读取屏幕上的辅助功能树，并发送键盘和鼠标事件。" + screen_recording: + name: "屏幕录制" + why: "为计算机使用观察结果截取窗口截图。" orchestrate: restore_default: "恢复默认" child_approval: