From 961d22cc0746e3e56dc5924adbe70c84b1059bad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Tue, 28 Apr 2026 17:15:28 -0400 Subject: [PATCH] fix: handle iOS keyboard Done dismiss controls --- AGENTS.md | 8 ++++-- .../RunnerTests+Interaction.swift | 28 +++++++++++++++++-- src/utils/__tests__/args.test.ts | 2 +- src/utils/command-schema.ts | 2 +- .../suites/agent-device-smoke-suite.ts | 13 +++++++++ website/docs/docs/commands.md | 2 +- website/docs/docs/introduction.md | 2 +- 7 files changed, 48 insertions(+), 9 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 968bb560d8..57f430d666 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -103,6 +103,7 @@ Command-only flags (like `find --first`) that don't flow to the platform layer o ## Hard Rules - Use `runCmd`/`runCmdSync` from `src/utils/exec.ts` for process execution. - Use daemon session flow for interactions (`open` before interactions, `close` after). +- Use `keyboard dismiss` for iOS keyboard dismissal; it may tap safe native controls such as `Done` but must not fall back to system back navigation. - Do not remove shared snapshot/session model behavior without full migration. - Command/device support must come from `src/core/capabilities.ts`. - Apple-family target changes must keep `src/utils/device.ts`, `src/core/capabilities.ts`, `src/core/dispatch-resolve.ts`, `src/platforms/ios/devices.ts`, and `src/platforms/ios/runner-xctestrun.ts` in sync. @@ -186,14 +187,15 @@ Command-only flags (like `find --first`) that don't flow to the platform layer o ## Docs & Skills - Versioned CLI help is the agent-facing source of truth. Put workflow guidance in `src/utils/command-schema.ts` help topics and assert important copy in `src/utils/__tests__/args.test.ts`. - Skills are thin routers. Keep `skills/**/SKILL.md` focused on when to use the skill, version gating, which `agent-device help ` page to read, and a short default loop. Do not duplicate full CLI manuals in skills. -- For behavior/CLI surface changes, update `README.md`, relevant `website/docs/**`, and router skills only when their short routing guidance or version assumptions change. -- For command-planning guidance changes, update `test/skillgym/suites/agent-device-smoke-suite.ts` when the change should alter what an agent plans. +- For behavior/CLI surface changes, update the versioned help instructions in `src/utils/command-schema.ts` and assert important help copy in `src/utils/__tests__/args.test.ts`. Also update `README.md` and relevant `website/docs/**` when user-facing docs need it. +- For behavior/CLI surface changes and command-planning guidance changes, write or update a SkillGym case in `test/skillgym/suites/agent-device-smoke-suite.ts` that captures the expected agent command plan. +- Do not update `skills/**/SKILL.md` for command behavior or workflow guidance unless the user explicitly asks; skills must route to versioned CLI help instead of carrying behavior details. - Keep SkillGym cases behavioral and command-planning oriented. Prefer prompts that assert the user-visible contract and expected command family over brittle exact output, but forbid known bad patterns. - Build before SkillGym when local CLI help is needed: `pnpm build`, then `pnpm exec skillgym run ... --case `. - Run SkillGym broad validation in batches of 20 cases or fewer using repeated `--case` runs; do not rely on one full-suite invocation for large runs. - Preserve current high-value workflow guidance: - iOS Expo Go dogfood: prefer `agent-device open "Expo Go" --platform ios` when the shell is known, then `snapshot -i` to confirm the project UI rather than the runner splash. - - `keyboard dismiss` is best-effort on iOS; prefer a visible app dismiss control, or `back --system` only when system navigation is acceptable. + - `keyboard dismiss` is the preferred iOS keyboard-dismissal path before manually pressing visible keyboard controls such as `Done`; it remains best-effort and can report unsupported layouts explicitly. - Empty replacement is not a supported clear-field command; do not document or test `fill ""` as clearing. Prefer visible clear/reset controls or report the tool gap. - Mutating commands against one session must run serially. Parallelize only read-only commands or commands on separate sessions/devices. - In final summaries, state whether docs/skills were updated; if not, explain why. diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 1997dfbcd0..25bd36e694 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -285,20 +285,44 @@ extension RunnerTests { } private func tapKeyboardDismissControl(app: XCUIApplication) -> Bool { - for label in ["Hide keyboard", "Dismiss keyboard"] { + let keyboardFrame = app.keyboards.firstMatch.frame + for label in ["Hide keyboard", "Dismiss keyboard", "Done"] { let candidates = [ app.keyboards.buttons[label], app.keyboards.keys[label], - app.toolbars.buttons[label], + app.keyboards.toolbars.buttons[label], ] if let hittable = candidates.first(where: { $0.exists && $0.isHittable }) { hittable.tap() return true } + + let toolbarButtonPredicate = NSPredicate( + format: "label == %@ OR identifier == %@", + label, + label + ) + let toolbarButtons = app.toolbars.buttons + .matching(toolbarButtonPredicate) + .allElementsBoundByIndex + if let hittable = toolbarButtons.first(where: { + $0.exists && $0.isHittable && isKeyboardAccessoryControl($0, keyboardFrame: keyboardFrame) + }) { + hittable.tap() + return true + } } return false } + private func isKeyboardAccessoryControl(_ element: XCUIElement, keyboardFrame: CGRect) -> Bool { + let frame = element.frame + guard !frame.isEmpty && !keyboardFrame.isEmpty else { + return false + } + return frame.intersects(keyboardFrame) || abs(frame.maxY - keyboardFrame.minY) <= 80 + } + private func moveCaretToEnd(element: XCUIElement) { let frame = element.frame guard !frame.isEmpty else { diff --git a/src/utils/__tests__/args.test.ts b/src/utils/__tests__/args.test.ts index fa0e256b94..fd8f86b29d 100644 --- a/src/utils/__tests__/args.test.ts +++ b/src/utils/__tests__/args.test.ts @@ -860,7 +860,7 @@ test('usageForCommand resolves workflow help topic', () => { assert.match(help, /iOS Allow Paste prompt cannot be exercised under XCUITest/); assert.match(help, /Empty replacement is not a supported clear-field command/); assert.match(help, /do not plan fill ""/); - assert.match(help, /iOS keyboard dismiss is best-effort/); + assert.match(help, /prefer keyboard dismiss before manually pressing visible Done/); assert.match(help, /UNSUPPORTED_OPERATION/); assert.match(help, /Stateful commands against one --session must run serially/); assert.match( diff --git a/src/utils/command-schema.ts b/src/utils/command-schema.ts index cbe9f6a0df..48e7921a22 100644 --- a/src/utils/command-schema.ts +++ b/src/utils/command-schema.ts @@ -287,7 +287,7 @@ Text entry: agent-device type "Handle with care" --delay-ms 80 Empty replacement is not a supported clear-field command: do not plan fill "" or fill ''. Prefer a visible clear/reset control; if the app exposes none, report the tool gap instead of inventing a clear command. Debounced field with no result selector: agent-device wait 1000. Keyboard read-only: keyboard status/get. Blocked control: try keyboard dismiss when supported. - iOS keyboard dismiss is best-effort and can return UNSUPPORTED_OPERATION when no native dismiss gesture/control is available. Prefer a visible app dismiss control, or use back --system only when system navigation is an acceptable side effect. + On iOS, prefer keyboard dismiss before manually pressing visible Done; the runner can use safe native keyboard controls and still reports unsupported layouts explicitly. If it returns UNSUPPORTED_OPERATION, prefer a visible app dismiss control, or use back --system only when system navigation is an acceptable side effect. Search-as-you-type fields on iOS can drop characters when driven too fast; use --delay-ms on fill/type before trying clipboard paste. iOS Allow Paste prompt cannot be exercised under XCUITest. To test paste-driven app behavior, prefill first with agent-device clipboard write "some text"; test the system prompt manually. Android non-ASCII can fail on some system images. Try fill/type normally; agent-device uses safer fallbacks. If the shell reports unsupported non-ASCII input, configure a trusted ADB keyboard IME outside the command plan and restore the previous IME afterward. diff --git a/test/skillgym/suites/agent-device-smoke-suite.ts b/test/skillgym/suites/agent-device-smoke-suite.ts index bc7b4db954..3eceb72d80 100644 --- a/test/skillgym/suites/agent-device-smoke-suite.ts +++ b/test/skillgym/suites/agent-device-smoke-suite.ts @@ -350,6 +350,19 @@ const FIXTURE_SMOKE_CASES: TestCase[] = [ outputs: [/field-name/i, /Done/i, commandAlternativesPattern(['press', 'click'])], forbiddenOutputs: [commandPattern('keyboard dismiss'), commandPattern('back')], }), + makeCase({ + id: 'form-keyboard-dismiss-ios-done-control', + contract: [ + 'Platform: iOS', + 'App name: Agent Device Tester', + 'Current screen: Checkout form tab', + 'testID=field-name', + 'The focused field shows an iOS keyboard toolbar with a visible Done control', + ], + task: 'Plan the commands to focus the Full name field and dismiss the iOS keyboard without manually pressing Done.', + outputs: [/field-name/i, /keyboard dismiss/i], + forbiddenOutputs: [commandPattern('back'), /press\s+.*Done/i, /click\s+.*Done/i], + }), makeCase({ id: 'form-reset', contract: [ diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 10b9d42530..90d565939b 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -534,7 +534,7 @@ agent-device keyboard dismiss ``` - `keyboard status` (or `keyboard get`) returns keyboard visibility and best-effort input type classification on Android. -- `keyboard dismiss` attempts a non-navigation keyboard dismissal on Android and a native dismiss gesture/control on iOS, then confirms the keyboard is hidden. +- `keyboard dismiss` attempts a non-navigation keyboard dismissal on Android and a native dismiss gesture/control on iOS, including common safe controls such as a keyboard toolbar `Done` button, then confirms the keyboard is hidden. - If the keyboard remains visible after the platform-native dismiss path, the command returns an explicit `UNSUPPORTED_OPERATION` error instead of falling back to back navigation. - On iOS, `keyboard dismiss` is best-effort and can fail when the active app exposes no native dismiss gesture/control. Prefer a visible app dismiss control, or use `back --system` only when system navigation is an acceptable side effect. - Works with active sessions and explicit selectors (`--platform`, `--device`, `--udid`, `--serial`). diff --git a/website/docs/docs/introduction.md b/website/docs/docs/introduction.md index 975676f7b7..14feaa4139 100644 --- a/website/docs/docs/introduction.md +++ b/website/docs/docs/introduction.md @@ -36,7 +36,7 @@ For agent-oriented operating guidance, start with `agent-device help` or `agent- - Physical-device recording defaults to 15 FPS and supports `--fps` caps. - `record start --quality <5-10>` scales recording resolution from 50% through native resolution; omitting it keeps native/current resolution. - Android supports the same core interaction set, plus `rotate`, `push` notification simulation, `clipboard read/write`, and `keyboard status|get|dismiss`. -- iOS `keyboard dismiss` is best-effort through the XCTest runner and can fail when the app exposes no native dismiss gesture/control. +- iOS `keyboard dismiss` is best-effort through the XCTest runner, including common native controls such as keyboard toolbar `Done`, and can fail when the app exposes no native dismiss gesture/control. - App-event triggers are available on iOS and Android through app-defined deep-link hooks (`trigger-app-event`), using active session context or explicit device selectors. ## Architecture (high level)