diff --git a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index c0bee3ca55..4bf5dc0f62 100644 --- a/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -1411,7 +1411,9 @@ extension RunnerTests { ok: false, error: ErrorPayload( code: "UNSUPPORTED_OPERATION", - message: "Unable to dismiss the iOS keyboard without a safe native dismiss control" + message: "Unable to dismiss the iOS keyboard without a safe native dismiss control", + hint: + "The on-screen keyboard usually does not block agent-device interactions: press the next target directly instead of retrying dismiss. If that press fails or reports no visible effect, scroll the target into view, or use keyboard enter to press the return key when submission is wanted." ) ) } diff --git a/src/cli/parser/__tests__/command-suggestions.test.ts b/src/cli/parser/__tests__/command-suggestions.test.ts index 7c2ffbf96f..1ed03effa1 100644 --- a/src/cli/parser/__tests__/command-suggestions.test.ts +++ b/src/cli/parser/__tests__/command-suggestions.test.ts @@ -97,6 +97,18 @@ test('input and settext suggest fill', () => { } }); +test('get-text, gettext, and get_text suggest get text', () => { + for (const guess of ['get-text', 'gettext', 'get_text']) { + assert.throws( + () => parseArgs([guess]), + (error: unknown) => + error instanceof AppError && + error.code === 'INVALID_ARGS' && + error.message === `Unknown command: ${guess}. Did you mean get text?`, + ); + } +}); + test('screencap and capture suggest screenshot', () => { for (const guess of ['screencap', 'capture']) { assert.throws( diff --git a/src/cli/parser/cli-help.ts b/src/cli/parser/cli-help.ts index 666cd71016..0570b3b0b0 100644 --- a/src/cli/parser/cli-help.ts +++ b/src/cli/parser/cli-help.ts @@ -168,7 +168,7 @@ Targets: Recovery: Network/typeahead result missing: wait text "Expected result" or wait . - Keyboard blocks target: keyboard dismiss when supported, then snapshot -i. + Keyboard visible over the next target: the on-screen keyboard usually does not block presses, so press the target directly instead of dismissing. If the press fails or reports no visible effect, scroll the target into view or use keyboard enter when submission is wanted. Sparse or recovered accessibility snapshot: use screenshot as visual truth, leave the bad screen if needed, then retry snapshot -i. Non-hittable success hint: verify with the settled diff or snapshot; retarget by a better ref/selector if the UI did not change.`, }, @@ -250,8 +250,8 @@ Text entry: agent-device press 'id="product-note"' agent-device type "Handle with care" --delay-ms 80 Empty replacement is not a supported clear-field command: do not plan fill "" or fill ''. Prefer a visible clear/reset control; if the app exposes none, report the tool gap instead of inventing a clear command. - Debounced field with no result selector: agent-device wait 1000. Keyboard read-only: keyboard status/get. Blocked control: try keyboard dismiss when supported. - To hide the keyboard, use keyboard dismiss. It taps safe controls like Done when available and verifies the keyboard closed. If it reports UNSUPPORTED_OPERATION, press a visible app control such as Done only when that is the intended fallback. + Debounced field with no result selector: agent-device wait 1000. Keyboard read-only: keyboard status/get. The on-screen keyboard usually does not block agent-device interactions; press the next target directly instead of dismissing. If that press fails or reports no visible effect, scroll the target into view or use keyboard enter when submission is wanted. + Only dismiss the keyboard when hiding it is the actual goal. To hide the keyboard, use keyboard dismiss. It taps safe controls like Done when available and verifies the keyboard closed. If it reports UNSUPPORTED_OPERATION, press a visible app control such as Done only when that is the intended fallback, or press the next target directly instead of retrying dismiss. Use plain fill/type first for ordinary login and form fields. If an iOS debounced or search-as-you-type field actually drops characters, or must receive incremental updates, retry with --delay-ms before trying clipboard paste; --delay-ms intentionally paces character entry. iOS Allow Paste prompt cannot be exercised under XCUITest. To test paste-driven app behavior, prefill first with agent-device clipboard write "some text"; test the system prompt manually. Android Gboard handwriting/stylus UI can capture text in an IME-owned input instead of the app field. If fill reports that input was captured by the keyboard/IME, use the diagnostic targetInput/actualInput details, inspect keyboard status/get if needed, and switch or disable handwriting outside the command plan before retrying. Do not keep retrying fill/type against the same field while the IME owns focus. diff --git a/src/cli/parser/command-suggestions.ts b/src/cli/parser/command-suggestions.ts index 67c059f5b5..fac4a52372 100644 --- a/src/cli/parser/command-suggestions.ts +++ b/src/cli/parser/command-suggestions.ts @@ -35,6 +35,9 @@ const COMMAND_ALIAS_SUGGESTIONS: Record = { screencap: { command: 'screenshot', example: 'screenshot' }, capture: { command: 'screenshot', example: 'screenshot' }, dismiss: { command: 'keyboard', example: 'keyboard dismiss' }, + 'get-text': { command: 'get', example: 'get text' }, + gettext: { command: 'get', example: 'get text' }, + get_text: { command: 'get', example: 'get text' }, }; export function listCommandAliasSuggestionEntries(): Array<[string, CommandAliasSuggestion]> { diff --git a/src/platforms/apple/core/__tests__/runner-client.test.ts b/src/platforms/apple/core/__tests__/runner-client.test.ts index af68193f17..949dcc6274 100644 --- a/src/platforms/apple/core/__tests__/runner-client.test.ts +++ b/src/platforms/apple/core/__tests__/runner-client.test.ts @@ -644,6 +644,39 @@ test('parseRunnerResponse preserves runner unsupported-operation codes', async ( ); }); +test('parseRunnerResponse surfaces the keyboard-dismiss hint to press the next target directly', async () => { + const hint = + 'The on-screen keyboard usually does not block agent-device interactions: press the next target directly instead of retrying dismiss. If that press fails or reports no visible effect, scroll the target into view, or use keyboard enter to press the return key when submission is wanted.'; + const response = new Response( + JSON.stringify({ + ok: false, + error: { + code: 'UNSUPPORTED_OPERATION', + message: 'Unable to dismiss the iOS keyboard without a safe native dismiss control', + hint, + }, + }), + ); + const session = { ready: false }; + + await assert.rejects( + () => parseRunnerResponse(response, session, '/tmp/runner.log'), + (error: unknown) => { + assert.ok(error instanceof AppError); + assert.equal(error.code, 'UNSUPPORTED_OPERATION'); + assert.equal(error.details?.hint, hint); + assert.match( + String(error.details?.hint), + /usually does not block agent-device interactions/i, + ); + assert.match(String(error.details?.hint), /press the next target directly/i); + assert.match(String(error.details?.hint), /scroll the target into view/i); + assert.match(String(error.details?.hint), /keyboard enter/i); + return true; + }, + ); +}); + test('parseRunnerResponse preserves iOS AX snapshot failure code and hint', async () => { const hint = 'Try a smaller read such as snapshot -s -d 8, or use direct selector commands such as find id click.';