diff --git a/src/commands/interaction/runtime/interactions.test.ts b/src/commands/interaction/runtime/interactions.test.ts index b0f8748511..e1b9a1c15b 100644 --- a/src/commands/interaction/runtime/interactions.test.ts +++ b/src/commands/interaction/runtime/interactions.test.ts @@ -127,6 +127,145 @@ test('runtime fill uses backend ref primitive without resolving snapshot geometr assert.deepEqual(result.backendResult, { ref: 'e1', text: 'hello' }); }); +test('native ref click preflight refuses an off-screen ref without calling the backend', async () => { + // Closed-drawer shape (ADR 0011): the stored session snapshot already holds + // the node, so the fast path must refuse it with the runtime path's exact + // offscreen_ref shape instead of letting the backend silently "succeed". + const calls: string[] = []; + const device = createInteractionDevice(offscreenDrawerSnapshot(), { + platform: 'web', + captureSnapshot: async () => { + throw new Error('native ref preflight must not capture a snapshot'); + }, + tapTarget: async (_context, target) => { + calls.push(target.ref); + return {}; + }, + }); + + await assert.rejects( + () => device.interactions.click(ref('@e2'), { session: 'default' }), + (error: unknown) => { + assert.ok(error instanceof Error); + assert.match(error.message, /Ref @e2 is off-screen and not safe to click/); + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, 'offscreen_ref'); + assert.equal(details?.ref, 'e2'); + assert.ok(typeof details?.hint === 'string'); + return true; + }, + ); + assert.deepEqual(calls, []); +}); + +test('native ref fill preflight refuses an off-screen ref without calling the backend', async () => { + const calls: string[] = []; + const device = createInteractionDevice(offscreenDrawerSnapshot(), { + platform: 'web', + captureSnapshot: async () => { + throw new Error('native ref preflight must not capture a snapshot'); + }, + fillTarget: async (_context, target) => { + calls.push(target.ref); + return {}; + }, + }); + + await assert.rejects( + () => device.interactions.fill(ref('@e2'), 'hello', { session: 'default' }), + (error: unknown) => { + assert.ok(error instanceof Error); + assert.match(error.message, /Ref @e2 is off-screen and not safe to fill/); + const details = (error as { details?: Record }).details; + assert.equal(details?.reason, 'offscreen_ref'); + return true; + }, + ); + assert.deepEqual(calls, []); +}); + +test('native ref click preflight refuses a covered ref without calling the backend', async () => { + const calls: string[] = []; + const device = createInteractionDevice(coveredByTabBarSnapshot(), { + platform: 'web', + captureSnapshot: async () => { + throw new Error('native ref preflight must not capture a snapshot'); + }, + tapTarget: async (_context, target) => { + calls.push(target.ref); + return {}; + }, + }); + + await assert.rejects( + () => device.interactions.click(ref('@e2'), { session: 'default' }), + (error: unknown) => { + assert.ok(error instanceof Error); + // Same shape as the runtime path's buildCoveredInteractionError. + assert.match(error.message, /Ref @e2 is covered by another visible element/); + const details = (error as { details?: Record }).details; + assert.equal(details?.ref, '@e2'); + assert.equal(details?.interactionBlocked, 'covered'); + return true; + }, + ); + assert.deepEqual(calls, []); +}); + +test('native ref click preflight annotates non-hittable targets but still calls the backend', async () => { + const calls: string[] = []; + const device = createInteractionDevice(nonHittableCellSnapshot(), { + platform: 'web', + captureSnapshot: async () => { + throw new Error('native ref preflight must not capture a snapshot'); + }, + tapTarget: async (_context, target) => { + calls.push(target.ref); + return { ref: target.ref.replace(/^@/, '') }; + }, + }); + + const result = await device.interactions.click(ref('@e2'), { session: 'default' }); + + // Annotation only: the backend still acts on the ref (no promotion on the + // fast path), and the result carries the same targetHittable/hint fields + // the runtime path attaches. + assert.deepEqual(calls, ['@e2']); + assert.equal(result.kind, 'ref'); + assert.equal(result.targetHittable, false); + assert.match(result.hint ?? '', /hittable: false/); + assert.deepEqual(result.backendResult, { ref: 'e2' }); +}); + +test('native ref fast path proceeds untouched when the session has no snapshot', async () => { + const calls: string[] = []; + const device = createAgentDevice({ + backend: { + platform: 'web', + captureSnapshot: async () => { + throw new Error('native ref preflight must not capture a snapshot'); + }, + tap: async () => {}, + typeText: async () => {}, + tapTarget: async (_context, target) => { + calls.push(target.ref); + return { ref: target.ref.replace(/^@/, '') }; + }, + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([{ name: 'default' }]), + policy: localCommandPolicy(), + }); + + const result = await device.interactions.click(ref('@e2'), { session: 'default' }); + + assert.deepEqual(calls, ['@e2']); + assert.equal(result.kind, 'ref'); + assert.equal(result.targetHittable, undefined); + assert.equal(result.hint, undefined); + assert.deepEqual(result.backendResult, { ref: 'e2' }); +}); + test('runtime interactions pass runtime signal to backend primitives', async () => { const controller = new AbortController(); let signal: AbortSignal | undefined; @@ -1254,6 +1393,29 @@ function selectorSnapshot(): SnapshotState { ]); } +// Closed-drawer shape shared by the native-ref preflight tests: the only +// interactive node (@e2) sits fully left of the Application viewport. +function offscreenDrawerSnapshot(): SnapshotState { + return makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 400, height: 800 }, + hittable: true, + }, + { + index: 1, + depth: 2, + parentIndex: 0, + type: 'Button', + label: 'Explore', + rect: { x: -320, y: 240, width: 300, height: 50 }, + hittable: true, + }, + ]); +} + function runtimeScrollSnapshot(options: { hiddenBelow: boolean; message?: string }): SnapshotState { return makeSnapshotState([ { diff --git a/src/commands/interaction/runtime/interactions.ts b/src/commands/interaction/runtime/interactions.ts index 958f0b4609..4f3d87e55b 100644 --- a/src/commands/interaction/runtime/interactions.ts +++ b/src/commands/interaction/runtime/interactions.ts @@ -24,6 +24,7 @@ import type { RepeatedInput } from '../../command-input.ts'; import { captureInteractionSnapshot, type InteractionTarget, + preflightNativeRefInteraction, resolveInteractionTarget, } from './resolution.ts'; @@ -272,6 +273,11 @@ async function maybeTapRefTarget( return null; } if (hasNonDefaultTapOptions(options)) return null; + // ADR 0011 native-ref preflight: the shared occlusion/offscreen guards run + // against the stored session snapshot node before the backend call (a + // backend fast path can silently "succeed", so errors must be raised here). + // No snapshot / no usable rect → no-op; never adds a capture round trip. + const preflight = await preflightNativeRefInteraction(runtime, options, options.target, action); const backendResult = await runtime.backend.tapTarget(toBackendContext(runtime, options), { kind: 'ref', ref: options.target.ref, @@ -281,6 +287,7 @@ async function maybeTapRefTarget( return { kind: 'ref', target: { kind: 'ref', ref: options.target.ref }, + ...preflight, ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), }; } @@ -290,6 +297,8 @@ async function maybeFillRefTarget( options: FillCommandOptions, ): Promise { if (options.target.kind !== 'ref' || !runtime.backend.fillTarget) return null; + // ADR 0011 native-ref preflight — see maybeTapRefTarget. + const preflight = await preflightNativeRefInteraction(runtime, options, options.target, 'fill'); const backendResult = await runtime.backend.fillTarget( toBackendContext(runtime, options), { @@ -305,6 +314,7 @@ async function maybeFillRefTarget( kind: 'ref', target: { kind: 'ref', ref: options.target.ref }, text: options.text, + ...preflight, ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), }; } diff --git a/src/commands/interaction/runtime/resolution.ts b/src/commands/interaction/runtime/resolution.ts index 81725eed6a..19be6e4ae9 100644 --- a/src/commands/interaction/runtime/resolution.ts +++ b/src/commands/interaction/runtime/resolution.ts @@ -443,6 +443,41 @@ function assertVisibleRefTarget( }); } +/** + * ADR 0011 native-ref preflight: `click @ref` / `fill @ref` fast paths + * dispatch straight to `backend.tapTarget`/`fillTarget`, and a backend fast + * path can silently "succeed" — delegation-on-error never triggers there. The + * ref came from the stored session snapshot, so the node is already in hand: + * run the SAME shared guards the runtime path uses against it before the + * backend call — occlusion (`isSnapshotNodeInteractionBlocked` via + * `assertInteractionNotBlocked`) and offscreen (`isNodeVisibleOnScreen` via + * `assertVisibleRefTarget`) ERROR with the runtime path's exact shapes, and + * the non-hittable annotation is returned for the fast-path result. + * + * Zero extra round trips by construction: no session, no stored snapshot, an + * unresolvable/invalid ref, or a node without a usable rect all make the + * preflight a no-op and the fast path proceeds exactly as before. Promotion + * to a hittable ancestor stays a runtime-path behavior — the preflight never + * changes which element the backend acts on. + */ +export async function preflightNativeRefInteraction( + runtime: AgentDeviceRuntime, + options: CommandContext, + target: Extract, + action: InteractionAction, +): Promise<{ targetHittable?: boolean; hint?: string }> { + const session = await runtime.sessions.get(options.session ?? 'default'); + const nodes = session?.snapshot?.nodes; + if (!nodes || normalizeRef(target.ref) === null) return {}; + const resolved = tryResolveRefNode(nodes, target.ref, { + fallbackLabel: target.fallbackLabel ?? '', + }); + if (!resolved) return {}; + assertInteractionNotBlocked(resolved.node, `Ref ${target.ref}`, action); + assertVisibleRefTarget(resolved.node, nodes, target.ref, action); + return describeNonHittableTarget(resolved.node, action); +} + // isNodeVisibleOnScreen (not the effective-viewport form): items inside an // off-screen scrollable container (closed drawer) must also count as // off-screen, not just items scrolled out of an on-screen container. diff --git a/src/contracts/__tests__/interaction-guarantees.test.ts b/src/contracts/__tests__/interaction-guarantees.test.ts index 2be2f660aa..a4babef588 100644 --- a/src/contracts/__tests__/interaction-guarantees.test.ts +++ b/src/contracts/__tests__/interaction-guarantees.test.ts @@ -189,9 +189,6 @@ test('acknowledged gaps are visible and bounded', () => { 'direct-ios-selector/responseIdentity', 'maestro-non-hittable-fallback/errorTaxonomy', 'maestro-non-hittable-fallback/responseConstruction', - 'native-ref/nonHittable', - 'native-ref/occlusion', - 'native-ref/offscreen', 'native-ref/responseConstruction', 'runtime-ref/responseConstruction', 'runtime-selector/responseConstruction', diff --git a/src/contracts/interaction-guarantees.ts b/src/contracts/interaction-guarantees.ts index 3281826bdd..4689823782 100644 --- a/src/contracts/interaction-guarantees.ts +++ b/src/contracts/interaction-guarantees.ts @@ -23,7 +23,8 @@ * where the fast path can succeed on a candidate the runtime rules would * refuse stay gaps until proven); and a shared runtime preflight against the * already-captured snapshot node for the native-ref path, because a backend - * fast path can silently succeed and delegation-on-error never triggers. + * fast path can silently succeed and delegation-on-error never triggers + * (implemented: preflightNativeRefInteraction, #1081). */ export const INTERACTION_GUARANTEES = [ @@ -256,7 +257,7 @@ export const INTERACTION_DISPATCH_PATHS: Record