diff --git a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift index d48ea57d50..7bf76ce7a9 100644 --- a/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +++ b/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift @@ -232,6 +232,13 @@ final class RunnerTests: XCTestCase { return Response(ok: true, data: DataPayload(message: "tapped")) } return Response(ok: false, error: ErrorPayload(message: "tap requires text or x/y")) + case .longPress: + guard let x = command.x, let y = command.y else { + return Response(ok: false, error: ErrorPayload(message: "longPress requires x and y")) + } + let duration = (command.durationMs ?? 800) / 1000.0 + longPressAt(app: activeApp, x: x, y: y, duration: duration) + return Response(ok: true, data: DataPayload(message: "long pressed")) case .type: guard let text = command.text else { return Response(ok: false, error: ErrorPayload(message: "type requires text")) @@ -411,6 +418,12 @@ final class RunnerTests: XCTestCase { coordinate.tap() } + private func longPressAt(app: XCUIApplication, x: Double, y: Double, duration: TimeInterval) { + let origin = app.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0)) + let coordinate = origin.withOffset(CGVector(dx: x, dy: y)) + coordinate.press(forDuration: duration) + } + private func swipe(app: XCUIApplication, direction: SwipeDirection) { let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app let start = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.2)) @@ -792,6 +805,7 @@ private func resolveRunnerPort() -> UInt16 { enum CommandType: String, Codable { case tap + case longPress case type case swipe case findText @@ -820,6 +834,7 @@ struct Command: Codable { let action: String? let x: Double? let y: Double? + let durationMs: Double? let direction: SwipeDirection? let scale: Double? let interactiveOnly: Bool? diff --git a/src/core/dispatch.ts b/src/core/dispatch.ts index 2e46d962cd..33e2b872ca 100644 --- a/src/core/dispatch.ts +++ b/src/core/dispatch.ts @@ -12,7 +12,7 @@ import { snapshotAndroid, } from '../platforms/android/index.ts'; import { listIosDevices } from '../platforms/ios/devices.ts'; -import { getInteractor } from '../utils/interactors.ts'; +import { getInteractor, type RunnerContext } from '../utils/interactors.ts'; import { runIosRunnerCommand } from '../platforms/ios/runner-client.ts'; import { snapshotAx } from '../platforms/ios/ax-snapshot.ts'; import { setIosSetting } from '../platforms/ios/index.ts'; @@ -92,7 +92,13 @@ export async function dispatchCommand( snapshotBackend?: 'ax' | 'xctest'; }, ): Promise | void> { - const interactor = getInteractor(device); + const runnerCtx: RunnerContext = { + appBundleId: context?.appBundleId, + verbose: context?.verbose, + logPath: context?.logPath, + traceLogPath: context?.traceLogPath, + }; + const interactor = getInteractor(device, runnerCtx); switch (command) { case 'open': { const app = positionals[0]; @@ -114,15 +120,7 @@ export async function dispatchCommand( case 'press': { const [x, y] = positionals.map(Number); if (Number.isNaN(x) || Number.isNaN(y)) throw new AppError('INVALID_ARGS', 'press requires x y'); - if (device.platform === 'ios' && device.kind === 'simulator') { - await runIosRunnerCommand( - device, - { command: 'tap', x, y, appBundleId: context?.appBundleId }, - { verbose: context?.verbose, logPath: context?.logPath, traceLogPath: context?.traceLogPath }, - ); - } else { - await interactor.tap(x, y); - } + await interactor.tap(x, y); return { x, y }; } case 'long-press': { @@ -138,29 +136,13 @@ export async function dispatchCommand( case 'focus': { const [x, y] = positionals.map(Number); if (Number.isNaN(x) || Number.isNaN(y)) throw new AppError('INVALID_ARGS', 'focus requires x y'); - if (device.platform === 'ios' && device.kind === 'simulator') { - await runIosRunnerCommand( - device, - { command: 'tap', x, y, appBundleId: context?.appBundleId }, - { verbose: context?.verbose, logPath: context?.logPath, traceLogPath: context?.traceLogPath }, - ); - } else { - await interactor.focus(x, y); - } + await interactor.focus(x, y); return { x, y }; } case 'type': { const text = positionals.join(' '); if (!text) throw new AppError('INVALID_ARGS', 'type requires text'); - if (device.platform === 'ios' && device.kind === 'simulator') { - await runIosRunnerCommand( - device, - { command: 'type', text, appBundleId: context?.appBundleId }, - { verbose: context?.verbose, logPath: context?.logPath, traceLogPath: context?.traceLogPath }, - ); - } else { - await interactor.type(text); - } + await interactor.type(text); return { text }; } case 'fill': { @@ -170,63 +152,21 @@ export async function dispatchCommand( if (Number.isNaN(x) || Number.isNaN(y) || !text) { throw new AppError('INVALID_ARGS', 'fill requires x y text'); } - if (device.platform === 'ios' && device.kind === 'simulator') { - await runIosRunnerCommand( - device, - { command: 'tap', x, y, appBundleId: context?.appBundleId }, - { verbose: context?.verbose, logPath: context?.logPath, traceLogPath: context?.traceLogPath }, - ); - await runIosRunnerCommand( - device, - { command: 'type', text, clearFirst: true, appBundleId: context?.appBundleId }, - { verbose: context?.verbose, logPath: context?.logPath, traceLogPath: context?.traceLogPath }, - ); - } else { - await interactor.fill(x, y, text); - } + await interactor.fill(x, y, text); return { x, y, text }; } case 'scroll': { const direction = positionals[0]; const amount = positionals[1] ? Number(positionals[1]) : undefined; if (!direction) throw new AppError('INVALID_ARGS', 'scroll requires direction'); - if (device.platform === 'ios' && device.kind === 'simulator') { - if (!['up', 'down', 'left', 'right'].includes(direction)) { - throw new AppError('INVALID_ARGS', `Unknown direction: ${direction}`); - } - const inverted = invertScrollDirection(direction as 'up' | 'down' | 'left' | 'right'); - await runIosRunnerCommand( - device, - { command: 'swipe', direction: inverted, appBundleId: context?.appBundleId }, - { verbose: context?.verbose, logPath: context?.logPath, traceLogPath: context?.traceLogPath }, - ); - } else { - await interactor.scroll(direction, amount); - } + await interactor.scroll(direction, amount); return { direction, amount }; } case 'scrollintoview': { const text = positionals.join(' ').trim(); if (!text) throw new AppError('INVALID_ARGS', 'scrollintoview requires text'); - if (device.platform === 'ios' && device.kind === 'simulator') { - const maxAttempts = 8; - for (let attempt = 0; attempt < maxAttempts; attempt += 1) { - const found = (await runIosRunnerCommand( - device, - { command: 'findText', text, appBundleId: context?.appBundleId }, - { verbose: context?.verbose, logPath: context?.logPath, traceLogPath: context?.traceLogPath }, - )) as { found?: boolean }; - if (found?.found) return { text, attempts: attempt + 1 }; - await runIosRunnerCommand( - device, - { command: 'swipe', direction: 'up', appBundleId: context?.appBundleId }, - { verbose: context?.verbose, logPath: context?.logPath, traceLogPath: context?.traceLogPath }, - ); - await new Promise((resolve) => setTimeout(resolve, 300)); - } - throw new AppError('COMMAND_FAILED', `scrollintoview could not find text: ${text}`); - } - await interactor.scrollIntoView(text); + const result = await interactor.scrollIntoView(text); + if (result?.attempts) return { text, attempts: result.attempts }; return { text }; } case 'pinch': { @@ -339,18 +279,3 @@ export async function dispatchCommand( throw new AppError('INVALID_ARGS', `Unknown command: ${command}`); } } - -function invertScrollDirection(direction: 'up' | 'down' | 'left' | 'right'): 'up' | 'down' | 'left' | 'right' { - switch (direction) { - case 'up': - return 'down'; - case 'down': - return 'up'; - case 'left': - return 'right'; - case 'right': - return 'left'; - } -} - -// Runner-only input on iOS simulators (simctl io input is not supported). diff --git a/src/platforms/ios/index.ts b/src/platforms/ios/index.ts index defc62680d..17d452e5ab 100644 --- a/src/platforms/ios/index.ts +++ b/src/platforms/ios/index.ts @@ -83,68 +83,6 @@ export async function closeIosApp(device: DeviceInfo, app: string): Promise { - ensureSimulator(device, 'press'); - throw new AppError( - 'UNSUPPORTED_OPERATION', - 'simctl io tap is not available; use the XCTest runner for input', - ); -} - -export async function longPressIos( - device: DeviceInfo, - _x: number, - _y: number, - _durationMs = 800, -): Promise { - ensureSimulator(device, 'long-press'); - throw new AppError( - 'UNSUPPORTED_OPERATION', - 'long-press is not supported on iOS simulators without XCTest runner support', - ); -} - -export async function focusIos(device: DeviceInfo, x: number, y: number): Promise { - await pressIos(device, x, y); -} - -export async function typeIos(device: DeviceInfo, _text: string): Promise { - ensureSimulator(device, 'type'); - throw new AppError( - 'UNSUPPORTED_OPERATION', - 'simctl io keyboard is not available; use the XCTest runner for input', - ); -} - -export async function fillIos( - device: DeviceInfo, - x: number, - y: number, - text: string, -): Promise { - await focusIos(device, x, y); - await typeIos(device, text); -} - -export async function scrollIos( - device: DeviceInfo, - _direction: string, - _amount = 0.6, -): Promise { - ensureSimulator(device, 'scroll'); - throw new AppError( - 'UNSUPPORTED_OPERATION', - 'simctl io swipe is not available; use the XCTest runner for input', - ); -} - -export async function scrollIntoViewIos(text: string): Promise { - throw new AppError( - 'UNSUPPORTED_OPERATION', - `scrollintoview is not supported on iOS without UI automation (${text})`, - ); -} - export async function screenshotIos(device: DeviceInfo, outPath: string): Promise { if (device.kind === 'simulator') { await ensureBootedSimulator(device); diff --git a/src/platforms/ios/runner-client.ts b/src/platforms/ios/runner-client.ts index 1b18c00b51..c9082d77bc 100644 --- a/src/platforms/ios/runner-client.ts +++ b/src/platforms/ios/runner-client.ts @@ -11,6 +11,7 @@ import net from 'node:net'; export type RunnerCommand = { command: | 'tap' + | 'longPress' | 'type' | 'swipe' | 'findText' @@ -27,6 +28,7 @@ export type RunnerCommand = { action?: 'get' | 'accept' | 'dismiss'; x?: number; y?: number; + durationMs?: number; direction?: 'up' | 'down' | 'left' | 'right'; scale?: number; interactiveOnly?: boolean; diff --git a/src/utils/interactors.ts b/src/utils/interactors.ts index 634c44137a..2d9e71eebc 100644 --- a/src/utils/interactors.ts +++ b/src/utils/interactors.ts @@ -15,17 +15,18 @@ import { } from '../platforms/android/index.ts'; import { closeIosApp, - fillIos, - focusIos, - longPressIos, openIosApp, openIosDevice, - pressIos, - scrollIos, - scrollIntoViewIos, screenshotIos, - typeIos, } from '../platforms/ios/index.ts'; +import { runIosRunnerCommand } from '../platforms/ios/runner-client.ts'; + +export type RunnerContext = { + appBundleId?: string; + verbose?: boolean; + logPath?: string; + traceLogPath?: string; +}; export type Interactor = { open(app: string, options?: { activity?: string }): Promise; @@ -37,11 +38,11 @@ export type Interactor = { type(text: string): Promise; fill(x: number, y: number, text: string): Promise; scroll(direction: string, amount?: number): Promise; - scrollIntoView(text: string): Promise; + scrollIntoView(text: string): Promise<{ attempts?: number } | void>; screenshot(outPath: string): Promise; }; -export function getInteractor(device: DeviceInfo): Interactor { +export function getInteractor(device: DeviceInfo, runnerContext: RunnerContext): Interactor { switch (device.platform) { case 'android': return { @@ -62,16 +63,101 @@ export function getInteractor(device: DeviceInfo): Interactor { open: (app) => openIosApp(device, app), openDevice: () => openIosDevice(device), close: (app) => closeIosApp(device, app), - tap: (x, y) => pressIos(device, x, y), - longPress: (x, y, durationMs) => longPressIos(device, x, y, durationMs), - focus: (x, y) => focusIos(device, x, y), - type: (text) => typeIos(device, text), - fill: (x, y, text) => fillIos(device, x, y, text), - scroll: (direction, amount) => scrollIos(device, direction, amount), - scrollIntoView: (text) => scrollIntoViewIos(text), screenshot: (outPath) => screenshotIos(device, outPath), + ...iosRunnerOverrides(device, runnerContext), }; default: throw new AppError('UNSUPPORTED_PLATFORM', `Unsupported platform: ${device.platform}`); } } + +type IoRunnerOverrides = Pick; + +function iosRunnerOverrides(device: DeviceInfo, ctx: RunnerContext): IoRunnerOverrides { + const runnerOpts = { verbose: ctx.verbose, logPath: ctx.logPath, traceLogPath: ctx.traceLogPath }; + + return { + tap: async (x, y) => { + await runIosRunnerCommand( + device, + { command: 'tap', x, y, appBundleId: ctx.appBundleId }, + runnerOpts, + ); + }, + longPress: async (x, y, durationMs) => { + await runIosRunnerCommand( + device, + { command: 'longPress', x, y, durationMs, appBundleId: ctx.appBundleId }, + runnerOpts, + ); + }, + focus: async (x, y) => { + await runIosRunnerCommand( + device, + { command: 'tap', x, y, appBundleId: ctx.appBundleId }, + runnerOpts, + ); + }, + type: async (text) => { + await runIosRunnerCommand( + device, + { command: 'type', text, appBundleId: ctx.appBundleId }, + runnerOpts, + ); + }, + fill: async (x, y, text) => { + await runIosRunnerCommand( + device, + { command: 'tap', x, y, appBundleId: ctx.appBundleId }, + runnerOpts, + ); + await runIosRunnerCommand( + device, + { command: 'type', text, clearFirst: true, appBundleId: ctx.appBundleId }, + runnerOpts, + ); + }, + scroll: async (direction, _amount) => { + if (!['up', 'down', 'left', 'right'].includes(direction)) { + throw new AppError('INVALID_ARGS', `Unknown direction: ${direction}`); + } + const inverted = invertScrollDirection(direction as 'up' | 'down' | 'left' | 'right'); + await runIosRunnerCommand( + device, + { command: 'swipe', direction: inverted, appBundleId: ctx.appBundleId }, + runnerOpts, + ); + }, + scrollIntoView: async (text) => { + const maxAttempts = 8; + for (let attempt = 0; attempt < maxAttempts; attempt += 1) { + const found = (await runIosRunnerCommand( + device, + { command: 'findText', text, appBundleId: ctx.appBundleId }, + runnerOpts, + )) as { found?: boolean }; + if (found?.found) return { attempts: attempt + 1 }; + await runIosRunnerCommand( + device, + { command: 'swipe', direction: 'up', appBundleId: ctx.appBundleId }, + runnerOpts, + ); + await new Promise((resolve) => setTimeout(resolve, 300)); + } + throw new AppError('COMMAND_FAILED', `scrollintoview could not find text: ${text}`); + }, + }; +} + +function invertScrollDirection(direction: 'up' | 'down' | 'left' | 'right'): 'up' | 'down' | 'left' | 'right' { + switch (direction) { + case 'up': + return 'down'; + case 'down': + return 'up'; + case 'left': + return 'right'; + case 'right': + return 'left'; + } +}