From f85293fb636718a570f78a9b4969be0bd17b19aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 3 Jul 2026 11:38:32 +0200 Subject: [PATCH 1/4] feat(interaction): opt-in --verify evidence for press/click/fill (#1047) Adds an opt-in --verify flag that returns cheap post-action evidence (foregroundApp, nodeCount, interactiveNodeCount, digest, changedFromBefore) instead of requiring a full follow-up snapshot to confirm a mutating command had an effect. The digest hashes the (type, label, identifier) multiset of an interactive-only capture, order-independent so it doesn't flip on harmless re-ordering; the node tree is never serialized back to the client, only the digest and counts. Default behavior (no --verify) is byte-identical to today. Implements the approved design from the #1047 issue comment: - src/utils/ax-digest.ts: new standalone digest module. - Pre-action digest reuses the snapshot the resolution path already captures for ref/selector targets (zero extra cost); point targets opt into one extra baseline capture only when --verify is set. - Post-action: one interactive-only capture through the same capture helper, digested and discarded. - --verify threaded through the CLI flag schema, MCP input schema, interactionResultExtra allowlist, and MCP output schemas, following the same plumbing as --double-tap and the #1040 targetHittable precedent. - The native-ref/direct-iOS-selector fast paths are skipped when --verify is set, since they bypass the resolution/capture path evidence depends on. --- src/cli/parser/cli-flags.ts | 9 + src/client/client-normalizers.ts | 1 + src/client/client-types.ts | 12 +- src/commands/interaction/index.ts | 9 +- src/commands/interaction/interactions.ts | 3 + src/commands/interaction/metadata.ts | 8 + .../interaction/runtime/interactions.test.ts | 171 ++++++++++++++++ .../interaction/runtime/interactions.ts | 58 +++++- .../interaction/runtime/resolution.ts | 40 +++- src/contracts/interaction.ts | 22 ++ .../handlers/__tests__/interaction.test.ts | 190 ++++++++++++++++++ .../handlers/interaction-touch-targets.ts | 8 +- src/daemon/handlers/interaction-touch.ts | 4 + src/mcp/command-output-schemas.ts | 22 +- src/utils/__tests__/ax-digest.test.ts | 82 ++++++++ src/utils/ax-digest.ts | 121 +++++++++++ 16 files changed, 748 insertions(+), 12 deletions(-) create mode 100644 src/utils/__tests__/ax-digest.test.ts create mode 100644 src/utils/ax-digest.ts diff --git a/src/cli/parser/cli-flags.ts b/src/cli/parser/cli-flags.ts index 92072b1bad..2645690fa1 100644 --- a/src/cli/parser/cli-flags.ts +++ b/src/cli/parser/cli-flags.ts @@ -105,6 +105,7 @@ export type CliFlags = CloudProviderProfileFields & jitterPx?: number; pixels?: number; doubleTap?: boolean; + verify?: boolean; clickButton?: ClickButton; backMode?: BackMode; pauseMs?: number; @@ -839,6 +840,14 @@ const FLAG_DEFINITIONS: readonly FlagDefinition[] = [ usageLabel: '--double-tap', usageDescription: 'Use double-tap gesture per press iteration', }, + { + key: 'verify', + names: ['--verify'], + type: 'boolean', + usageLabel: '--verify', + usageDescription: + 'Capture cheap post-action evidence (AX digest, node counts, changedFromBefore) instead of a follow-up snapshot', + }, { key: 'clickButton', names: ['--button'], diff --git a/src/client/client-normalizers.ts b/src/client/client-normalizers.ts index f5608933a1..f47dc0dfa3 100644 --- a/src/client/client-normalizers.ts +++ b/src/client/client-normalizers.ts @@ -342,6 +342,7 @@ export function buildFlags(options: InternalRequestOptions): CommandFlags { jitterPx: options.jitterPx, pixels: options.pixels, doubleTap: options.doubleTap, + verify: options.verify, clickButton: options.clickButton, pauseMs: options.pauseMs, pattern: options.pattern, diff --git a/src/client/client-types.ts b/src/client/client-types.ts index cf1bb13ca5..dc146c9c7a 100644 --- a/src/client/client-types.ts +++ b/src/client/client-types.ts @@ -613,12 +613,20 @@ export type ClickOptions = DeviceCommandBaseOptions & InteractionTarget & RepeatedPressOptions & { button?: ClickButton; + /** + * Opt-in (#1047): return cheap post-action evidence (AX digest, node counts, + * changedFromBefore) in the response instead of requiring a follow-up + * snapshot to confirm the action had an effect. + */ + verify?: boolean; }; export type PressOptions = DeviceCommandBaseOptions & SelectorSnapshotCommandOptions & InteractionTarget & - RepeatedPressOptions; + RepeatedPressOptions & { + verify?: boolean; + }; export type LongPressOptions = DeviceCommandBaseOptions & SelectorSnapshotCommandOptions & @@ -671,6 +679,7 @@ export type FillOptions = DeviceCommandBaseOptions & InteractionTarget & { text: string; delayMs?: number; + verify?: boolean; }; export type ScrollOptions = DeviceCommandBaseOptions & { @@ -897,6 +906,7 @@ type CommandExecutionOptions = Partial & { jitterPx?: number; pixels?: number; doubleTap?: boolean; + verify?: boolean; clickButton?: ClickButton; pauseMs?: number; pattern?: SwipePattern; diff --git a/src/commands/interaction/index.ts b/src/commands/interaction/index.ts index 411e5f44d4..6ae015fb33 100644 --- a/src/commands/interaction/index.ts +++ b/src/commands/interaction/index.ts @@ -70,7 +70,7 @@ const interactionCliSchemas = { usageOverride: 'click ', positionalArgs: ['target'], allowsExtraPositionals: true, - allowedFlags: [...REPEATED_TOUCH_FLAGS, 'clickButton', ...SELECTOR_SNAPSHOT_FLAGS], + allowedFlags: [...REPEATED_TOUCH_FLAGS, 'clickButton', 'verify', ...SELECTOR_SNAPSHOT_FLAGS], }, press: { usageOverride: 'press ', @@ -78,7 +78,7 @@ const interactionCliSchemas = { 'Short press a semantic UI target by ref, selector, or point. For native context menus or hold gestures, use longpress instead of press --hold-ms.', positionalArgs: ['targetOrX', 'y?'], allowsExtraPositionals: true, - allowedFlags: [...REPEATED_TOUCH_FLAGS, ...SELECTOR_SNAPSHOT_FLAGS], + allowedFlags: [...REPEATED_TOUCH_FLAGS, 'verify', ...SELECTOR_SNAPSHOT_FLAGS], }, longpress: { usageOverride: 'longpress [durationMs]', @@ -114,7 +114,7 @@ const interactionCliSchemas = { usageOverride: 'fill | fill <@ref|selector> ', positionalArgs: ['targetOrX', 'yOrText', 'text?'], allowsExtraPositionals: true, - allowedFlags: [...SELECTOR_SNAPSHOT_FLAGS, 'delayMs'], + allowedFlags: [...SELECTOR_SNAPSHOT_FLAGS, 'delayMs', 'verify'], }, scroll: { usageOverride: 'scroll [amount] [--pixels ] [--duration-ms ]', @@ -342,6 +342,7 @@ function toClickOptions(input: ClickInput): ClickOptions { ...toSelectorSnapshotOptions(input), ...toRepeatedOptions(input), button: input.button, + verify: input.verify, }; } @@ -351,6 +352,7 @@ function toPressOptions(input: PressInput): PressOptions { ...toClientInteractionTarget(input.target), ...toSelectorSnapshotOptions(input), ...toRepeatedOptions(input), + verify: input.verify, }; } @@ -361,6 +363,7 @@ function toFillOptions(input: FillInput): FillOptions { ...toSelectorSnapshotOptions(input), text: input.text, delayMs: input.delayMs, + verify: input.verify, }; } diff --git a/src/commands/interaction/interactions.ts b/src/commands/interaction/interactions.ts index 2864c20912..f5a1453398 100644 --- a/src/commands/interaction/interactions.ts +++ b/src/commands/interaction/interactions.ts @@ -37,12 +37,14 @@ export const interactionCliReaders = { ...repeatedInputFromFlags(flags), target: targetInputFromClientTarget(readInteractionTargetFromPositionals(positionals)), button: flags.clickButton, + verify: flags.verify, }), press: (positionals, flags) => ({ ...commonInputFromFlags(flags), ...selectorSnapshotInputFromFlags(flags), ...repeatedInputFromFlags(flags), target: targetInputFromClientTarget(readInteractionTargetFromPositionals(positionals)), + verify: flags.verify, }), longpress: (positionals, flags) => { const decoded = readLongPressTargetFromPositionals(positionals); @@ -80,6 +82,7 @@ export const interactionCliReaders = { target: targetInputFromClientTarget(decoded.target), text: decoded.text, delayMs: flags.delayMs, + verify: flags.verify, }; }, scroll: (positionals, flags) => ({ diff --git a/src/commands/interaction/metadata.ts b/src/commands/interaction/metadata.ts index 1f3f71455a..ad322f6af4 100644 --- a/src/commands/interaction/metadata.ts +++ b/src/commands/interaction/metadata.ts @@ -66,17 +66,24 @@ const interactionCommandDescriptions = { type InteractionCommandName = keyof typeof interactionCommandDescriptions; +const verifyField = () => + booleanField( + 'Capture cheap post-action evidence (AX digest, node counts, changedFromBefore) instead of a follow-up snapshot.', + ); + const clickFields = { target: requiredField(interactionTargetField()), button: enumField(CLICK_BUTTONS, 'Pointer button for platforms that support mouse buttons.'), ...selectorSnapshotFields(), ...repeatedFields(), + verify: verifyField(), }; const pressFields = { target: requiredField(interactionTargetField()), ...selectorSnapshotFields(), ...repeatedFields(), + verify: verifyField(), }; const fillFields = { @@ -84,6 +91,7 @@ const fillFields = { text: requiredField(stringField('Text to enter into the target.')), delayMs: integerField('Delay between typed characters.', { min: 0 }), ...selectorSnapshotFields(), + verify: verifyField(), }; const longPressFields = { diff --git a/src/commands/interaction/runtime/interactions.test.ts b/src/commands/interaction/runtime/interactions.test.ts index ed53026943..bd4400a7ae 100644 --- a/src/commands/interaction/runtime/interactions.test.ts +++ b/src/commands/interaction/runtime/interactions.test.ts @@ -27,6 +27,56 @@ test('runtime click taps an explicit point without requiring a snapshot', async assert.deepEqual(result, { kind: 'point', point: { x: 10, y: 20 } }); }); +test('runtime click with verify captures a baseline for point targets and reports evidence', async () => { + let captureCount = 0; + const device = createInteractionDevice(selectorSnapshot(), { + captureSnapshot: async () => { + captureCount += 1; + if (captureCount === 1) return { snapshot: selectorSnapshot() }; + return { snapshot: makeSnapshotState([]) }; + }, + tap: async () => ({ ok: true }), + }); + + const result = await device.interactions.click( + { kind: 'point', x: 10, y: 20 }, + { session: 'default', verify: true }, + ); + + assert.equal(result.kind, 'point'); + assert.ok(result.evidence); + assert.equal(result.evidence?.changedFromBefore, true); + assert.equal(result.evidence?.nodeCount, 0); + assert.equal(captureCount, 2); +}); + +test('runtime click with verify skips the native ref fast path so evidence can be captured', async () => { + const calls: string[] = []; + let captureCount = 0; + const device = createInteractionDevice(selectorSnapshot(), { + platform: 'web', + captureSnapshot: async () => { + captureCount += 1; + return { snapshot: selectorSnapshot() }; + }, + tapTarget: async (_context, target) => { + calls.push(target.ref); + return { ref: target.ref.replace(/^@/, '') }; + }, + tap: async () => ({ ok: true }), + }); + + const result = await device.interactions.click(ref('@e1'), { + session: 'default', + verify: true, + }); + + assert.deepEqual(calls, []); + assert.equal(result.kind, 'ref'); + assert.ok(result.evidence); + assert.ok(captureCount >= 1); +}); + test('runtime click uses backend ref primitive without resolving snapshot geometry', async () => { const calls: string[] = []; const device = createInteractionDevice(selectorSnapshot(), { @@ -162,6 +212,127 @@ test('runtime selector interactions fall back to a full snapshot when interactiv ]); }); +test('runtime press without verify omits evidence entirely', async () => { + const device = createInteractionDevice(selectorSnapshot(), { + tap: async () => ({ ok: true }), + }); + + const result = await device.interactions.press(selector('label=Continue'), { + session: 'default', + }); + + assert.equal('evidence' in result, false); +}); + +test('runtime press with verify reports unchanged evidence when the post-action capture matches', async () => { + const device = createInteractionDevice(selectorSnapshot(), { + tap: async () => ({ ok: true }), + }); + + const result = await device.interactions.press(selector('label=Continue'), { + session: 'default', + verify: true, + }); + + assert.equal(result.kind, 'selector'); + assert.ok(result.evidence); + assert.equal(result.evidence?.changedFromBefore, false); + assert.equal(result.evidence?.nodeCount, 1); + assert.equal(result.evidence?.interactiveNodeCount, 1); + assert.equal(typeof result.evidence?.digest, 'string'); + assert.ok(result.evidence?.digest.startsWith('ax1:')); +}); + +test('runtime press with verify reports changedFromBefore true when the post-action capture differs', async () => { + let captureCount = 0; + const device = createInteractionDevice(selectorSnapshot(), { + captureSnapshot: async () => { + captureCount += 1; + if (captureCount === 1) return { snapshot: selectorSnapshot() }; + return { + snapshot: makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Button', + label: 'Continue', + value: 'Continue', + rect: { x: 10, y: 20, width: 100, height: 40 }, + hittable: true, + }, + { + index: 1, + depth: 0, + type: 'Text', + label: 'Loading…', + rect: { x: 10, y: 80, width: 100, height: 20 }, + hittable: true, + }, + ]), + }; + }, + tap: async () => ({ ok: true }), + }); + + const result = await device.interactions.press(selector('label=Continue'), { + session: 'default', + verify: true, + }); + + assert.equal(result.kind, 'selector'); + assert.ok(result.evidence); + assert.equal(result.evidence?.changedFromBefore, true); + assert.equal(result.evidence?.nodeCount, 2); +}); + +test('runtime fill without verify omits evidence entirely', async () => { + const device = createInteractionDevice(fillableSnapshot(), { + fill: async () => ({ ok: true }), + }); + + const result = await device.interactions.fill(selector('label=Email'), 'hi', { + session: 'default', + }); + + assert.equal('evidence' in result, false); +}); + +test('runtime fill with verify reports evidence and detects a changed post-action capture', async () => { + let captureCount = 0; + const device = createInteractionDevice(fillableSnapshot(), { + captureSnapshot: async () => { + captureCount += 1; + if (captureCount === 1) return { snapshot: fillableSnapshot() }; + return { + snapshot: makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'XCUIElementTypeTextField', + label: 'Email', + value: 'hi', + rect: { x: 20, y: 10, width: 60, height: 40 }, + hittable: true, + }, + ]), + }; + }, + fill: async () => ({ ok: true }), + }); + + const result = await device.interactions.fill(selector('label=Email'), 'hi', { + session: 'default', + verify: true, + }); + + assert.equal(result.kind, 'selector'); + assert.ok(result.evidence); + // Digest is over (type, label, identifier) only, so a value-only change does + // not flip the digest — this is intentional (see ax-digest.ts docs). + assert.equal(result.evidence?.changedFromBefore, false); + assert.equal(result.evidence?.nodeCount, 1); +}); + test('runtime click keeps distinct tab button centers when iOS reports the tab bar as hittable', async () => { const calls: Point[] = []; const device = createInteractionDevice(iosTabBarSnapshot(), { diff --git a/src/commands/interaction/runtime/interactions.ts b/src/commands/interaction/runtime/interactions.ts index 3ce59a4739..0d922d1c92 100644 --- a/src/commands/interaction/runtime/interactions.ts +++ b/src/commands/interaction/runtime/interactions.ts @@ -2,13 +2,16 @@ import { AppError } from '../../../kernel/errors.ts'; import type { ClickButton } from '../../../core/click-button.ts'; import type { AgentDeviceRuntime, CommandContext } from '../../../runtime-contract.ts'; import { isFillableType } from '../../../snapshot/snapshot-processing.ts'; -import type { Point } from '../../../kernel/snapshot.ts'; +import type { Point, SnapshotNode } from '../../../kernel/snapshot.ts'; import { requireIntInRange } from '../../../utils/validation.ts'; import { successText } from '../../../utils/success-text.ts'; import { findMistargetedTypeRefToken } from '../../../utils/type-target-warning.ts'; +import { summarizeAxEvidence } from '../../../utils/ax-digest.ts'; import type { FillCommandResult, + InteractionEvidence, PressCommandResult, + ResolvedInteractionTarget, ResolvedTarget, } from '../../../contracts/interaction.ts'; import { toBackendContext } from '../../runtime-common.ts'; @@ -18,7 +21,11 @@ import { type RuntimeCommand, } from '../../runtime-types.ts'; import type { RepeatedInput } from '../../command-input.ts'; -import { type InteractionTarget, resolveInteractionTarget } from './resolution.ts'; +import { + captureInteractionSnapshot, + type InteractionTarget, + resolveInteractionTarget, +} from './resolution.ts'; export { focusCommand, @@ -48,6 +55,12 @@ export type PressCommandOptions = CommandContext & RepeatedInput & { target: InteractionTarget; button?: ClickButton; + /** + * Opt-in (#1047): take one post-action interactive-only capture, digest it, + * and return it as `evidence` instead of the caller having to spend a full + * follow-up snapshot round trip to confirm the action had an effect. + */ + verify?: boolean; }; export type ClickCommandOptions = PressCommandOptions; @@ -58,6 +71,7 @@ export type FillCommandOptions = CommandContext & { target: InteractionTarget; text: string; delayMs?: number; + verify?: boolean; }; export type TypeTextCommandOptions = CommandContext & { @@ -86,13 +100,15 @@ export const fillCommand: RuntimeCommand options, ): Promise => { if (!options.text) throw new AppError('INVALID_ARGS', 'fill requires text'); - const nativeRefFill = await maybeFillRefTarget(runtime, options); + const verify = options.verify === true; + const nativeRefFill = verify ? null : await maybeFillRefTarget(runtime, options); if (nativeRefFill) return nativeRefFill; const resolved = await resolveInteractionTarget(runtime, options, { action: 'fill', requireInteractive: true, promoteToHittableAncestor: false, + captureEvidenceBaseline: verify, }); if (!runtime.backend.fill) { throw new AppError('UNSUPPORTED_OPERATION', 'fill is not supported by this backend'); @@ -110,11 +126,13 @@ export const fillCommand: RuntimeCommand nodeType && !isFillableType(nodeType, runtime.backend.platform) ? `fill target ${formatTargetForWarning(resolved)} resolved to "${nodeType}", attempting fill anyway.` : undefined; + const evidence = verify ? await captureVerifyEvidence(runtime, options, resolved) : undefined; return { ...resolved, text: options.text, ...(warning ? { warning } : {}), ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), + ...(evidence ? { evidence } : {}), }; }; @@ -156,13 +174,15 @@ async function tapCommand( options: PressCommandOptions, action: 'click' | 'press', ): Promise { - const nativeRefTap = await maybeTapRefTarget(runtime, options, action); + const verify = options.verify === true; + const nativeRefTap = verify ? null : await maybeTapRefTarget(runtime, options, action); if (nativeRefTap) return nativeRefTap; const resolved = await resolveInteractionTarget(runtime, options, { action, requireInteractive: true, promoteToHittableAncestor: true, + captureEvidenceBaseline: verify, }); if (!runtime.backend.tap) { throw new AppError('UNSUPPORTED_OPERATION', 'tap is not supported by this backend'); @@ -177,12 +197,42 @@ async function tapCommand( doubleTap: options.doubleTap, }); const formattedBackendResult = toBackendResult(backendResult); + const evidence = verify ? await captureVerifyEvidence(runtime, options, resolved) : undefined; return { ...resolved, ...(formattedBackendResult ? { backendResult: formattedBackendResult } : {}), + ...(evidence ? { evidence } : {}), }; } +/** + * Post-action side of `--verify` (#1047): one interactive-only capture through + * the same capture helper the resolution path already uses, digested and then + * discarded — the node tree itself is never attached to the result, only the + * cheap summary. Best-effort: a failed capture must not turn a successful + * action into a failure, so this returns `undefined` instead of throwing. + */ +async function captureVerifyEvidence( + runtime: AgentDeviceRuntime, + options: CommandContext, + resolved: ResolvedInteractionTarget, +): Promise { + const preActionNodes: SnapshotNode[] | undefined = + 'preActionNodes' in resolved ? resolved.preActionNodes : undefined; + try { + const capture = await captureInteractionSnapshot(runtime, options, true); + const after = summarizeAxEvidence(capture.snapshot.nodes); + // No pre-action baseline (for example the baseline capture itself failed) + // means we cannot claim a change happened — default to false rather than + // asserting a change we did not actually observe. + const changedFromBefore = + preActionNodes !== undefined && after.digest !== summarizeAxEvidence(preActionNodes).digest; + return { ...after, changedFromBefore }; + } catch { + return undefined; + } +} + function requireResolvedPoint(result: { point?: Point }): Point { if (!result.point) { throw new AppError('COMMAND_FAILED', 'Interaction target resolved without coordinates'); diff --git a/src/commands/interaction/runtime/resolution.ts b/src/commands/interaction/runtime/resolution.ts index a27adf0834..9f7fd7b7eb 100644 --- a/src/commands/interaction/runtime/resolution.ts +++ b/src/commands/interaction/runtime/resolution.ts @@ -40,6 +40,15 @@ type ResolveInteractionTargetParams = { action: InteractionAction; requireInteractive: boolean; promoteToHittableAncestor: boolean; + /** + * `--verify` (#1047): also capture the pre-action node set for a `point` target + * so `changedFromBefore` evidence has a baseline. Ref/selector targets already + * capture a snapshot to resolve the target, so this is a no-op cost for them — + * their nodes are attached below regardless of this flag. For point targets, + * which normally skip capture entirely, this opts into one extra capture, only + * when the caller explicitly asked for verify evidence. Defaults to false. + */ + captureEvidenceBaseline?: boolean; }; export async function resolveInteractionTarget( @@ -50,7 +59,7 @@ export async function resolveInteractionTarget( await assertSupportedInteractionSurface(runtime, options, params.action); if (options.target.kind === 'point') { - return resolvePointInteractionTarget(options.target); + return await resolvePointInteractionTarget(runtime, options, options.target, params); } if (options.target.kind === 'ref') { @@ -60,13 +69,38 @@ export async function resolveInteractionTarget( return await resolveSelectorInteractionTarget(runtime, options, options.target, params); } -function resolvePointInteractionTarget(target: PointTarget): ResolvedInteractionTarget { +async function resolvePointInteractionTarget( + runtime: AgentDeviceRuntime, + options: CommandContext, + target: PointTarget, + params: ResolveInteractionTargetParams, +): Promise { + if (!params.captureEvidenceBaseline) { + return { kind: 'point', point: { x: target.x, y: target.y } }; + } + const preActionNodes = await tryCaptureEvidenceBaseline(runtime, options); return { kind: 'point', point: { x: target.x, y: target.y }, + ...(preActionNodes ? { preActionNodes } : {}), }; } +async function tryCaptureEvidenceBaseline( + runtime: AgentDeviceRuntime, + options: CommandContext, +): Promise { + try { + const capture = await captureInteractionSnapshot(runtime, options, true); + return capture.snapshot.nodes; + } catch { + // Evidence is best-effort: a failed baseline capture must not fail the + // action itself. Post-action evidence (if any) will simply omit + // changedFromBefore. + return undefined; + } +} + async function resolveRefInteractionTarget( runtime: AgentDeviceRuntime, options: CommandContext, @@ -94,6 +128,7 @@ async function resolveRefInteractionTarget( }), refLabel: resolveRefLabel(node, capture.snapshot.nodes), ...describeNonHittableTarget(node, params.action), + preActionNodes: capture.snapshot.nodes, }; } @@ -161,6 +196,7 @@ async function resolveSelectorInteractionTarget( }), refLabel: resolveRefLabel(node, capture.snapshot.nodes), ...describeNonHittableTarget(node, params.action), + preActionNodes: capture.snapshot.nodes, }; } diff --git a/src/contracts/interaction.ts b/src/contracts/interaction.ts index 673c010aa9..0116403064 100644 --- a/src/contracts/interaction.ts +++ b/src/contracts/interaction.ts @@ -35,6 +35,7 @@ export type ResolvedInteractionTarget = | { kind: 'point'; point: Point; + preActionNodes?: SnapshotNode[]; } | { kind: 'ref'; @@ -45,6 +46,7 @@ export type ResolvedInteractionTarget = refLabel?: string; targetHittable?: boolean; hint?: string; + preActionNodes?: SnapshotNode[]; } | { kind: 'selector'; @@ -55,11 +57,30 @@ export type ResolvedInteractionTarget = refLabel?: string; targetHittable?: boolean; hint?: string; + preActionNodes?: SnapshotNode[]; }; +/** + * Opt-in (`--verify`) cheap post-condition evidence for mutating interaction + * commands (#1047). `digest`/`nodeCount`/`interactiveNodeCount` describe a single + * interactive-only capture taken right after the action; `changedFromBefore` + * compares that digest against the pre-action capture the resolution path already + * held, so no extra device round trip is spent beyond the one verify capture. + * `changedFromBefore: false` is evidence, not failure — the command still + * succeeded. + */ +export type InteractionEvidence = { + foregroundApp?: string; + nodeCount: number; + interactiveNodeCount: number; + digest: string; + changedFromBefore: boolean; +}; + export type PressCommandResult = ResolvedInteractionTarget & { backendResult?: Record; message?: string; + evidence?: InteractionEvidence; }; export type FillCommandResult = ResolvedInteractionTarget & { @@ -67,6 +88,7 @@ export type FillCommandResult = ResolvedInteractionTarget & { warning?: string; backendResult?: Record; message?: string; + evidence?: InteractionEvidence; }; export type LongPressCommandResult = ResolvedInteractionTarget & { diff --git a/src/daemon/handlers/__tests__/interaction.test.ts b/src/daemon/handlers/__tests__/interaction.test.ts index 0db4597556..231ab05a30 100644 --- a/src/daemon/handlers/__tests__/interaction.test.ts +++ b/src/daemon/handlers/__tests__/interaction.test.ts @@ -1689,6 +1689,196 @@ test('press @ref fails when Android tap escapes to Settings', async () => { }); }); +test('press @ref --verify surfaces evidence through the interactionResultExtra allowlist', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'verify-press'; + const session = makeSession(sessionName); + session.snapshot = { + nodes: attachRefs([ + { + index: 0, + type: 'XCUIElementTypeButton', + label: 'Continue', + rect: { x: 10, y: 20, width: 100, height: 40 }, + enabled: true, + hittable: true, + }, + ]), + createdAt: Date.now(), + backend: 'xctest', + }; + sessionStore.set(sessionName, session); + + mockDispatch.mockImplementation(async (_device, command) => { + if (command === 'snapshot') { + // Post-action capture reports an extra node, so changedFromBefore should + // read true against the pre-action (stored) snapshot's single node. + return { + nodes: [ + { + index: 0, + type: 'XCUIElementTypeButton', + label: 'Continue', + rect: { x: 10, y: 20, width: 100, height: 40 }, + enabled: true, + hittable: true, + }, + { + index: 1, + type: 'XCUIElementTypeStaticText', + label: 'Loaded', + rect: { x: 10, y: 80, width: 100, height: 20 }, + enabled: true, + hittable: true, + }, + ], + backend: 'xctest', + }; + } + return { pressed: true }; + }); + + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'press', + positionals: ['@e1'], + flags: { verify: true }, + }, + sessionName, + sessionStore, + contextFromFlags, + }); + + expect(response?.ok).toBe(true); + if (response?.ok) { + const evidence = response.data?.evidence as + | { + nodeCount: number; + interactiveNodeCount: number; + digest: string; + changedFromBefore: boolean; + } + | undefined; + expect(evidence).toBeTruthy(); + expect(evidence?.nodeCount).toBe(2); + expect(evidence?.interactiveNodeCount).toBe(2); + expect(typeof evidence?.digest).toBe('string'); + expect(evidence?.changedFromBefore).toBe(true); + } + // The stored ref snapshot already had a valid rect, so resolution reused it + // without a fresh pre-action capture (zero extra cost, per #1047's design) — + // only the post-action verify capture issues a 'snapshot' dispatch, after 'press'. + expect(mockDispatch.mock.calls.map((call) => call[1])).toEqual(['press', 'snapshot']); +}); + +test('press @ref without --verify never includes an evidence field', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'no-verify-press'; + const session = makeSession(sessionName); + session.snapshot = { + nodes: attachRefs([ + { + index: 0, + type: 'XCUIElementTypeButton', + label: 'Continue', + rect: { x: 10, y: 20, width: 100, height: 40 }, + enabled: true, + hittable: true, + }, + ]), + createdAt: Date.now(), + backend: 'xctest', + }; + sessionStore.set(sessionName, session); + mockDispatch.mockResolvedValue({ pressed: true }); + + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'press', + positionals: ['@e1'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + }); + + expect(response?.ok).toBe(true); + if (response?.ok) { + expect(response.data?.evidence).toBeUndefined(); + } + // No verify flag means no post-action snapshot capture at all. + expect(mockDispatch.mock.calls.map((call) => call[1])).toEqual(['press']); +}); + +test('fill selector --verify surfaces evidence through the interactionResultExtra allowlist', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'verify-fill'; + const session = makeSession(sessionName); + session.snapshot = { + nodes: attachRefs([ + { + index: 0, + type: 'XCUIElementTypeTextField', + label: 'Email', + rect: { x: 10, y: 20, width: 100, height: 40 }, + enabled: true, + hittable: true, + }, + ]), + createdAt: Date.now(), + backend: 'xctest', + }; + sessionStore.set(sessionName, session); + + mockDispatch.mockImplementation(async (_device, command) => { + if (command === 'snapshot') { + return { + nodes: [ + { + index: 0, + type: 'XCUIElementTypeTextField', + label: 'Email', + rect: { x: 10, y: 20, width: 100, height: 40 }, + enabled: true, + hittable: true, + }, + ], + backend: 'xctest', + }; + } + return {}; + }); + + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'fill', + positionals: ['label=Email', 'hello@example.com'], + flags: { verify: true }, + }, + sessionName, + sessionStore, + contextFromFlags, + }); + + expect(response?.ok).toBe(true); + if (response?.ok) { + const evidence = response.data?.evidence as + | { nodeCount: number; changedFromBefore: boolean } + | undefined; + expect(evidence).toBeTruthy(); + expect(evidence?.nodeCount).toBe(1); + // Same node set before and after, so no change is reported. + expect(evidence?.changedFromBefore).toBe(false); + } +}); + test('press @ref promotes a non-hittable node to its hittable ancestor before tapping', async () => { const sessionStore = makeSessionStore(); const sessionName = 'default'; diff --git a/src/daemon/handlers/interaction-touch-targets.ts b/src/daemon/handlers/interaction-touch-targets.ts index 65d1fae9ee..8935f65ae5 100644 --- a/src/daemon/handlers/interaction-touch-targets.ts +++ b/src/daemon/handlers/interaction-touch-targets.ts @@ -127,6 +127,10 @@ export function parseFillTarget(positionals: string[]): ParsedFillTarget { export function interactionResultExtra( result: PressCommandResult | FillCommandResult | LongPressCommandResult, ): Record { + // `evidence` (#1047, opt-in via --verify) is additive on press/fill only — + // LongPressCommandResult has no evidence field, so it reads as undefined + // (and gets dropped by the response layer) for longpress. + const evidence = 'evidence' in result ? result.evidence : undefined; if (result.kind === 'ref') { return { ref: stripAtPrefix(result.target?.kind === 'ref' ? result.target.ref : undefined), @@ -134,6 +138,7 @@ export function interactionResultExtra( selectorChain: result.selectorChain, targetHittable: result.targetHittable, hint: result.hint, + evidence, }; } if (result.kind === 'selector') { @@ -143,9 +148,10 @@ export function interactionResultExtra( refLabel: result.refLabel, targetHittable: result.targetHittable, hint: result.hint, + evidence, }; } - return {}; + return { evidence }; } export function formatTouchTargetLabel( diff --git a/src/daemon/handlers/interaction-touch.ts b/src/daemon/handlers/interaction-touch.ts index 853af170e7..9c9df2e0dc 100644 --- a/src/daemon/handlers/interaction-touch.ts +++ b/src/daemon/handlers/interaction-touch.ts @@ -208,6 +208,7 @@ async function runTargetedTouchInteraction(params: { holdMs: flags?.holdMs, jitterPx: flags?.jitterPx, doubleTap: flags?.doubleTap, + verify: flags?.verify, }; return command === 'click' ? await runtime.interactions.click(target, options) @@ -259,6 +260,7 @@ function readDirectIosSelectorTapTarget(params: { if (commandLabel !== 'click') return null; if (target.kind !== 'selector') return null; if (hasNonDefaultClickOptions(flags)) return null; + if (flags?.verify === true) return null; const selector = readSimpleIosSelectorTarget({ session, selectorExpression: target.selector }); if (!selector) return null; return { @@ -443,6 +445,7 @@ async function dispatchFillViaRuntime( session: sessionName, requestId: req.meta?.requestId, delayMs: req.flags?.delayMs, + verify: req.flags?.verify, }), buildPayloads: (result) => { const referenceFrame = @@ -483,6 +486,7 @@ function readDirectIosSelectorFillTarget(params: { }): DirectIosSelectorTarget | null { const { session, target, flags } = params; if (target.kind !== 'selector') return null; + if (flags?.verify === true) return null; const selector = readSimpleIosSelectorTarget({ session, selectorExpression: target.selector }); if (!selector) return null; return { diff --git a/src/mcp/command-output-schemas.ts b/src/mcp/command-output-schemas.ts index b408044086..1528dc5a83 100644 --- a/src/mcp/command-output-schemas.ts +++ b/src/mcp/command-output-schemas.ts @@ -165,6 +165,21 @@ function interactionResultSchema(extra: InteractionExtra = {}): JsonSchema { const backendResultSchema = looseObjectSchema('Raw backend result passthrough.'); +// InteractionEvidence (src/contracts/interaction.ts) — opt-in `--verify` cheap +// post-condition evidence (#1047). +const interactionEvidenceSchema: JsonSchema = objectSchema( + { + foregroundApp: stringSchema('Foreground app bundle id or name, when the capture carries it.'), + nodeCount: numberSchema('Node count in the post-action interactive-only capture.'), + interactiveNodeCount: numberSchema('Subset of nodeCount the platform reports as hittable.'), + digest: stringSchema('Order-independent digest of the post-action node multiset.'), + changedFromBefore: booleanSchema( + 'Whether the post-action digest differs from the pre-action capture digest. false is evidence, not failure.', + ), + }, + ['nodeCount', 'interactiveNodeCount', 'digest', 'changedFromBefore'], +); + // boot / shutdown share the resolved-device header (src/contracts/device.ts). const deviceHeaderProperties: Record = { platform: enumSchema(PLATFORMS), @@ -190,7 +205,11 @@ const targetShutdownResultSchema: JsonSchema = objectSchema( export const COMMAND_OUTPUT_SCHEMAS = { // src/contracts/interaction.ts press: interactionResultSchema({ - properties: { backendResult: backendResultSchema, message: stringSchema() }, + properties: { + backendResult: backendResultSchema, + message: stringSchema(), + evidence: interactionEvidenceSchema, + }, }), fill: interactionResultSchema({ properties: { @@ -198,6 +217,7 @@ export const COMMAND_OUTPUT_SCHEMAS = { warning: stringSchema(), backendResult: backendResultSchema, message: stringSchema(), + evidence: interactionEvidenceSchema, }, required: ['text'], }), diff --git a/src/utils/__tests__/ax-digest.test.ts b/src/utils/__tests__/ax-digest.test.ts new file mode 100644 index 0000000000..4677240603 --- /dev/null +++ b/src/utils/__tests__/ax-digest.test.ts @@ -0,0 +1,82 @@ +import { expect, test } from 'vitest'; +import { computeAxDigest } from '../ax-digest.ts'; + +test('digest is stable across repeated calls for the same nodes', () => { + const nodes = [ + { type: 'button', label: 'Continue', identifier: 'continue-btn' }, + { type: 'text', label: 'Welcome' }, + ]; + + expect(computeAxDigest(nodes)).toEqual(computeAxDigest(nodes)); +}); + +test('digest is order-independent over the node multiset', () => { + const a = [ + { type: 'button', label: 'Continue', identifier: 'continue-btn' }, + { type: 'text', label: 'Welcome' }, + { type: 'image', label: 'Logo' }, + ]; + const b = [a[2]!, a[0]!, a[1]!]; + + expect(computeAxDigest(a).digest).toBe(computeAxDigest(b).digest); +}); + +test('digest changes when a node label changes', () => { + const before = computeAxDigest([{ type: 'button', label: 'Continue' }]); + const after = computeAxDigest([{ type: 'button', label: 'Continue!' }]); + + expect(after.digest).not.toBe(before.digest); +}); + +test('digest changes when a node type changes', () => { + const before = computeAxDigest([{ type: 'button', label: 'Continue' }]); + const after = computeAxDigest([{ type: 'link', label: 'Continue' }]); + + expect(after.digest).not.toBe(before.digest); +}); + +test('digest changes when a node identifier changes', () => { + const before = computeAxDigest([{ type: 'button', label: 'Continue', identifier: 'a' }]); + const after = computeAxDigest([{ type: 'button', label: 'Continue', identifier: 'b' }]); + + expect(after.digest).not.toBe(before.digest); +}); + +test('digest changes when node count changes even with the same multiset otherwise', () => { + const one = computeAxDigest([{ type: 'button', label: 'Continue' }]); + const two = computeAxDigest([ + { type: 'button', label: 'Continue' }, + { type: 'button', label: 'Continue' }, + ]); + + expect(two.digest).not.toBe(one.digest); + expect(two.nodeCount).toBe(2); + expect(one.nodeCount).toBe(1); +}); + +test('digest for an empty node array is stable and reports zero nodes', () => { + const result = computeAxDigest([]); + + expect(result.nodeCount).toBe(0); + expect(result.digest).toBe(computeAxDigest([]).digest); +}); + +test('digest is prefixed for forward-compatible versioning', () => { + const result = computeAxDigest([{ type: 'button', label: 'Continue' }]); + + expect(result.digest.startsWith('ax1:')).toBe(true); +}); + +test('digest ignores volatile fields such as rects that are not part of the tuple', () => { + const withRect = computeAxDigest([ + { + type: 'button', + label: 'Continue', + identifier: 'a', + ...({ rect: { x: 1, y: 2, width: 3, height: 4 } } as Record), + }, + ]); + const withoutRect = computeAxDigest([{ type: 'button', label: 'Continue', identifier: 'a' }]); + + expect(withRect.digest).toBe(withoutRect.digest); +}); diff --git a/src/utils/ax-digest.ts b/src/utils/ax-digest.ts new file mode 100644 index 0000000000..4f0d5731ef --- /dev/null +++ b/src/utils/ax-digest.ts @@ -0,0 +1,121 @@ +import { createHash } from 'node:crypto'; + +/** + * Stable, order-independent digest over an accessibility node array. + * + * Used to give agents cheap post-action evidence (see #1047) without paying the + * token cost of a full follow-up snapshot: the daemon computes this digest from a + * capture it already has to take, and returns only the ~50-byte digest string plus + * a couple of counts instead of the serialized node tree. + * + * The digest is a multiset hash over each node's (type, label, identifier) tuple — + * order-independent so it does not flip on harmless re-ordering (for example list + * virtualization or AX tree re-traversal), but sensitive to any node being added, + * removed, or relabeled. It intentionally ignores rects/indices/other volatile + * fields so it doesn't false-positive on scroll offsets or layout jitter alone. + * + * Combination is done with XOR (commutative, so node order never matters) over a + * fixed-size per-node hash, then folded through one more hash together with the + * node count so two different multisets that happen to XOR to the same value + * (extremely unlikely, but XOR alone is not collision-safe) still produce + * different digests. + */ + +export type AxDigestNode = { + type?: string; + label?: string; + identifier?: string; +}; + +export type AxDigestResult = { + digest: string; + nodeCount: number; +}; + +const DIGEST_PREFIX = 'ax1:'; +const PER_NODE_HASH_BYTES = 16; + +export function computeAxDigest(nodes: readonly AxDigestNode[]): AxDigestResult { + const combined = Buffer.alloc(PER_NODE_HASH_BYTES); + for (const node of nodes) { + xorInPlace(combined, hashNode(node)); + } + const finalized = createHash('sha256') + .update(combined) + .update('\0') + .update(String(nodes.length)) + .digest('hex') + .slice(0, 16); + return { digest: `${DIGEST_PREFIX}${finalized}`, nodeCount: nodes.length }; +} + +function hashNode(node: AxDigestNode): Buffer { + return createHash('sha256') + .update(node.type ?? '') + .update('\0') + .update(node.label ?? '') + .update('\0') + .update(node.identifier ?? '') + .digest() + .subarray(0, PER_NODE_HASH_BYTES); +} + +function xorInPlace(target: Buffer, other: Buffer): void { + for (let i = 0; i < target.length; i += 1) { + target[i] = (target[i] ?? 0) ^ (other[i] ?? 0); + } +} + +/** + * Node shape the evidence helpers below need beyond the digest tuple: `hittable` + * to derive `interactiveNodeCount`, and `bundleId`/`appName` to derive + * `foregroundApp` — both already present on `SnapshotNode` when the platform + * backend reports them, so reading them here costs nothing extra. + */ +export type AxEvidenceNode = AxDigestNode & { + hittable?: boolean; + bundleId?: string; + appName?: string; +}; + +export type AxEvidenceSummary = { + digest: string; + nodeCount: number; + interactiveNodeCount: number; + foregroundApp?: string; +}; + +/** + * Summarizes one capture into the pieces `evidence` needs (see + * src/contracts/interaction.ts: `InteractionEvidence`), without ever requiring + * the caller to serialize the node array itself. `interactiveNodeCount` counts + * nodes the platform did not mark `hittable: false` within the given capture — + * cheap since it's a filter over nodes already in hand, no extra signal needed. + * `foregroundApp` is only populated when the capture already carries an app + * scope on its nodes (`bundleId`/`appName`); this never triggers a separate + * appstate lookup. + */ +export function summarizeAxEvidence(nodes: readonly AxEvidenceNode[]): AxEvidenceSummary { + const { digest, nodeCount } = computeAxDigest(nodes); + const interactiveNodeCount = nodes.reduce( + (count, node) => (node.hittable === false ? count : count + 1), + 0, + ); + const foregroundApp = resolveForegroundApp(nodes); + return { + digest, + nodeCount, + interactiveNodeCount, + ...(foregroundApp ? { foregroundApp } : {}), + }; +} + +function resolveForegroundApp(nodes: readonly AxEvidenceNode[]): string | undefined { + for (const node of nodes) { + if (node.bundleId) return node.bundleId; + } + for (const node of nodes) { + if (node.appName) return node.appName; + } + return undefined; +} From 04202d9bb9274bf5e293dd04a575bf016de86f8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 3 Jul 2026 12:39:49 +0200 Subject: [PATCH 2/4] test: cover press --verify with a provider scenario; classify the flag CI's architecture-progress gate failed on 1 unclassified public flag. --verify drives real device captures, so it belongs in the device-observable list backed by an actual provider scenario rather than the intentionally-outside bucket: the new scenario asserts evidence (changedFromBefore, digest, nodeCount) on press @ref --verify, that the verify capture's tree is never serialized into the response, and that the transcript completes (snapshot -> tap -> verify snapshot; the @ref path reuses the session snapshot as its baseline, so no extra resolution capture entry exists). --- scripts/integration-progress-model.ts | 1 + .../interaction-verify.test.ts | 113 ++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 test/integration/provider-scenarios/interaction-verify.test.ts diff --git a/scripts/integration-progress-model.ts b/scripts/integration-progress-model.ts index 89c58d7f49..e6c870b2b7 100644 --- a/scripts/integration-progress-model.ts +++ b/scripts/integration-progress-model.ts @@ -184,6 +184,7 @@ function summarizeProviderScenarioFlagCoverage(files) { ['batchMaxSteps', 'batch max-step guard', ['maxSteps']], ['findFirst', 'find first disambiguation'], ['findLast', 'find last disambiguation'], + ['verify', 'post-action evidence capture on press/click/fill'], ]; const sources = files.map((file) => fs.readFileSync(file, 'utf8')).join('\n'); return flagTargets.map(([key, reason, aliases = []]) => { diff --git a/test/integration/provider-scenarios/interaction-verify.test.ts b/test/integration/provider-scenarios/interaction-verify.test.ts new file mode 100644 index 0000000000..c296c9742d --- /dev/null +++ b/test/integration/provider-scenarios/interaction-verify.test.ts @@ -0,0 +1,113 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { assertRpcOk } from './assertions.ts'; +import { PROVIDER_SCENARIO_IOS_SIMULATOR } from './fixtures.ts'; +import { createProviderScenarioHarness, withProviderScenarioResource } from './harness.ts'; +import { + createAppleRunnerProviderFromTranscript, + createRecordingAppleToolProvider, + simctlListDevicesHandler, +} from './providers.ts'; +import { createProviderTranscript } from './transcript.ts'; + +const APP = 'com.example.app'; +const DEVICE_ID = PROVIDER_SCENARIO_IOS_SIMULATOR.id; + +const BEFORE_NODES = [ + { + index: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 400, height: 800 }, + }, + { + index: 1, + parentIndex: 0, + type: 'Button', + label: 'Continue', + hittable: true, + rect: { x: 100, y: 300, width: 200, height: 44 }, + }, +]; + +const AFTER_NODES = [ + { + index: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 400, height: 800 }, + }, + { + index: 1, + parentIndex: 0, + type: 'StaticText', + label: 'Welcome!', + rect: { x: 100, y: 300, width: 200, height: 44 }, + }, +]; + +function snapshotEntry(nodes: unknown[]) { + return { + command: 'ios.runner.snapshot', + deviceId: DEVICE_ID, + platform: 'apple', + result: { nodes, truncated: false }, + }; +} + +test('Provider-backed integration press --verify returns post-action evidence digest', async () => { + const runnerTranscript = createProviderTranscript([ + // snapshot -i to obtain refs + snapshotEntry(BEFORE_NODES), + { + command: 'ios.runner.tap', + deviceId: DEVICE_ID, + platform: 'apple', + result: { x: 200, y: 322 }, + }, + // post-action verify capture: digested server-side, never serialized + snapshotEntry(AFTER_NODES), + ]); + const appleRunnerProvider = createAppleRunnerProviderFromTranscript( + runnerTranscript, + 'ios.runner', + ); + const appleTool = createRecordingAppleToolProvider({ + simctl: simctlListDevicesHandler('com.apple.CoreSimulator.SimRuntime.iOS-18-0', [ + { name: PROVIDER_SCENARIO_IOS_SIMULATOR.name, udid: DEVICE_ID }, + ]), + }); + + await withProviderScenarioResource( + async () => + await createProviderScenarioHarness({ + appleRunnerProvider: () => appleRunnerProvider, + appleToolProvider: () => appleTool.provider, + deviceInventoryProvider: async () => [PROVIDER_SCENARIO_IOS_SIMULATOR], + }), + async (daemon) => { + const open = await daemon.callCommand('open', [APP], { + platform: 'ios', + udid: DEVICE_ID, + }); + assertRpcOk(open); + + const snapshot = await daemon.callCommand('snapshot', [], { + snapshotInteractiveOnly: true, + }); + assertRpcOk(snapshot); + + const press = await daemon.callCommand('press', ['@e1'], { verify: true }); + const data = assertRpcOk(press); + const evidence = data.evidence as Record | undefined; + assert.ok(evidence, 'press --verify must return evidence'); + assert.equal(evidence.changedFromBefore, true); + assert.equal(typeof evidence.digest, 'string'); + assert.equal(evidence.nodeCount, AFTER_NODES.length); + // The verify capture's tree must never be serialized into the response. + assert.equal(data.nodes, undefined); + + runnerTranscript.assertComplete(); + }, + ); +}); From 70b719348000eb21bf2fba80047695641de7d339 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Fri, 3 Jul 2026 12:41:29 +0200 Subject: [PATCH 3/4] fix: type the verify scenario transcript entries --- .../integration/provider-scenarios/interaction-verify.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/provider-scenarios/interaction-verify.test.ts b/test/integration/provider-scenarios/interaction-verify.test.ts index c296c9742d..78fa9c9957 100644 --- a/test/integration/provider-scenarios/interaction-verify.test.ts +++ b/test/integration/provider-scenarios/interaction-verify.test.ts @@ -8,7 +8,7 @@ import { createRecordingAppleToolProvider, simctlListDevicesHandler, } from './providers.ts'; -import { createProviderTranscript } from './transcript.ts'; +import { createProviderTranscript, type ProviderScenarioProviderEntry } from './transcript.ts'; const APP = 'com.example.app'; const DEVICE_ID = PROVIDER_SCENARIO_IOS_SIMULATOR.id; @@ -46,7 +46,7 @@ const AFTER_NODES = [ }, ]; -function snapshotEntry(nodes: unknown[]) { +function snapshotEntry(nodes: unknown[]): ProviderScenarioProviderEntry { return { command: 'ios.runner.snapshot', deviceId: DEVICE_ID, From 8f49ce8adf54fc3ed1d13c11b23a094e5065ce5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sat, 4 Jul 2026 10:26:47 +0200 Subject: [PATCH 4/4] fix: include interaction extras in the fill @ref response branch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ref branch of dispatchFillViaRuntime rebuilt responseData from backendResult/coordinates, dropping interactionResultExtra(result) — so fill @ref --verify returned no evidence even though the post-action capture ran (live E2E gap found in PR #1064 review). Spreading the extras also gives fill @ref the same ref/refLabel/selectorChain (and conditional targetHittable/hint) fields press @ref already returns. Adds daemon tests for fill @ref --verify evidence and for the no-verify path staying evidence-free with no post-action capture. --- .../handlers/__tests__/interaction.test.ts | 114 ++++++++++++++++++ src/daemon/handlers/interaction-touch.ts | 4 + 2 files changed, 118 insertions(+) diff --git a/src/daemon/handlers/__tests__/interaction.test.ts b/src/daemon/handlers/__tests__/interaction.test.ts index 231ab05a30..edd5271dfc 100644 --- a/src/daemon/handlers/__tests__/interaction.test.ts +++ b/src/daemon/handlers/__tests__/interaction.test.ts @@ -1879,6 +1879,120 @@ test('fill selector --verify surfaces evidence through the interactionResultExtr } }); +test('fill @ref --verify surfaces evidence in the ref response branch', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'verify-fill-ref'; + const session = makeSession(sessionName); + session.snapshot = { + nodes: attachRefs([ + { + index: 0, + type: 'XCUIElementTypeTextField', + label: 'Email', + rect: { x: 10, y: 20, width: 100, height: 40 }, + enabled: true, + hittable: true, + }, + ]), + createdAt: Date.now(), + backend: 'xctest', + }; + sessionStore.set(sessionName, session); + + mockDispatch.mockImplementation(async (_device, command) => { + if (command === 'snapshot') { + return { + nodes: [ + { + index: 0, + type: 'XCUIElementTypeTextField', + label: 'Email', + rect: { x: 10, y: 20, width: 100, height: 40 }, + enabled: true, + hittable: true, + }, + { + index: 1, + type: 'XCUIElementTypeButton', + label: 'Submit', + rect: { x: 10, y: 80, width: 100, height: 40 }, + enabled: true, + hittable: true, + }, + ], + backend: 'xctest', + }; + } + return {}; + }); + + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'fill', + positionals: ['@e1', 'hello@example.com'], + flags: { verify: true }, + }, + sessionName, + sessionStore, + contextFromFlags, + }); + + expect(response?.ok).toBe(true); + if (response?.ok) { + const evidence = response.data?.evidence as + | { nodeCount: number; changedFromBefore: boolean; digest: string } + | undefined; + expect(evidence).toBeTruthy(); + expect(evidence?.nodeCount).toBe(2); + expect(evidence?.changedFromBefore).toBe(true); + expect(typeof evidence?.digest).toBe('string'); + } +}); + +test('fill @ref without --verify never includes an evidence field', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'no-verify-fill-ref'; + const session = makeSession(sessionName); + session.snapshot = { + nodes: attachRefs([ + { + index: 0, + type: 'XCUIElementTypeTextField', + label: 'Email', + rect: { x: 10, y: 20, width: 100, height: 40 }, + enabled: true, + hittable: true, + }, + ]), + createdAt: Date.now(), + backend: 'xctest', + }; + sessionStore.set(sessionName, session); + mockDispatch.mockResolvedValue({}); + + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'fill', + positionals: ['@e1', 'hello@example.com'], + flags: {}, + }, + sessionName, + sessionStore, + contextFromFlags, + }); + + expect(response?.ok).toBe(true); + if (response?.ok) { + expect(response.data?.evidence).toBeUndefined(); + } + // No verify flag means no post-action snapshot capture at all. + expect(mockDispatch.mock.calls.map((call) => call[1])).not.toContain('snapshot'); +}); + test('press @ref promotes a non-hittable node to its hittable ancestor before tapping', async () => { const sessionStore = makeSessionStore(); const sessionName = 'default'; diff --git a/src/daemon/handlers/interaction-touch.ts b/src/daemon/handlers/interaction-touch.ts index 9c9df2e0dc..e9462d5554 100644 --- a/src/daemon/handlers/interaction-touch.ts +++ b/src/daemon/handlers/interaction-touch.ts @@ -471,6 +471,10 @@ async function dispatchFillViaRuntime( ref: stripAtPrefix(result.target?.kind === 'ref' ? result.target.ref : undefined), ...(result.point ? { x: result.point.x, y: result.point.y } : {}), }), + // Same extras press @ref already returns — without this the ref + // branch rebuilt the response from backendResult and dropped + // evidence, so fill @ref --verify returned none (PR #1064 review). + ...interactionResultExtra(result), } : recordedResult; if (result.warning) responseData.warning = result.warning;