From b99b05b1d9bfe2d7d9794677a3ff58d45a099f37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 9 Jul 2026 12:05:59 +0200 Subject: [PATCH 1/2] feat: include unchanged interactive refs in settle output Benchmarks (gpt-5.4-mini + claude-haiku, July 2026) showed 27% of --settle actions were followed by a fallback snapshot -i because a change-only diff omits refs for elements that did not change: after a modal dismiss the diff shows only removals, so the next button to press is invisible. Add an unchanged-interactive tail to SettleObservation, attached only when the diff's added lines carry zero refs (the modal-dismiss/toast-only signature). It lists the settled tree's remaining hittable, uncovered elements so the response stays actionable without an extra round trip. Rides the CLI text, MCP digest view, ref pinning, and output schema the same way the diff's added-line refs already do. --- src/cli/parser/cli-help.ts | 2 +- src/commands/interaction/output.test.ts | 64 +++++ src/commands/interaction/output.ts | 25 +- .../interaction/runtime/settle.test.ts | 241 +++++++++++++++++- src/commands/interaction/runtime/settle.ts | 64 ++++- src/contracts/interaction.ts | 26 ++ .../__tests__/interaction-settle.test.ts | 68 +++++ src/daemon/response-views.ts | 9 +- src/mcp/__tests__/command-tools.test.ts | 36 +++ src/mcp/command-output-schemas.ts | 14 + src/mcp/command-tools.ts | 11 +- .../settle-observation.test.ts | 131 ++++++++++ 12 files changed, 679 insertions(+), 12 deletions(-) diff --git a/src/cli/parser/cli-help.ts b/src/cli/parser/cli-help.ts index 1a13e38eb0..666cd71016 100644 --- a/src/cli/parser/cli-help.ts +++ b/src/cli/parser/cli-help.ts @@ -223,7 +223,7 @@ Snapshots and refs: Anti-pattern: snapshot -i followed by snapshot -i | grep ..., or adding 2>/dev/null | jq ... before reading the raw command output. Refs from the first snapshot remain valid until you press, click, fill, type, scroll, go back, wait for async UI, or otherwise change app state. Pinned refs (@e12~s4, generation from refsGeneration or settle.refsGeneration) get exact staleness warnings instead of the coarse tree-changed one; plain refs stay valid input. - After a mutation, prefer a known selector/label directly (for example press 'label="Send"') because interaction commands refresh interactive state internally. If you need to discover a new control not shown by settle, use snapshot -i, or snapshot -i -s "Composer" when a stable container label/id can scope the refresh. + After a mutation, prefer a known selector/label directly (for example press 'label="Send"') because interaction commands refresh interactive state internally. A settled diff with no added refs (for example a modal dismiss) also lists an "unchanged interactive" tail of still-present refs, so check that before falling back. If you need to discover a new control not shown by settle or its tail, use snapshot -i, or snapshot -i -s "Composer" when a stable container label/id can scope the refresh. If typing/fill opened the keyboard or changed layout and the next target has no stable selector, run snapshot -i, use the fresh ref, then verify with wait/find or diff snapshot -i. For a targeted query, use find/get/is. If you truly need the full tree again, pass --force-full. Off-screen summaries are scroll hints; use scroll, not swipe, then snapshot -i. diff --git a/src/commands/interaction/output.test.ts b/src/commands/interaction/output.test.ts index fbdd5e6c6b..4b18bdca19 100644 --- a/src/commands/interaction/output.test.ts +++ b/src/commands/interaction/output.test.ts @@ -81,6 +81,70 @@ describe('press CLI output', () => { ); }); + test('appends the unchanged interactive tail after a removals-only diff', () => { + const output = formatPress({ + message: 'Tapped @e4 (100, 200)', + x: 100, + y: 200, + settle: { + settled: true, + waitedMs: 500, + diff: { + summary: { additions: 0, removals: 2, unchanged: 3 }, + lines: [ + { kind: 'removed', text: '@e4 [button] "OK"' }, + { kind: 'removed', text: '@e5 [text] "Are you sure?"' }, + ], + }, + tail: [ + { ref: 'e9', role: 'button', label: 'Add to cart' }, + { ref: 'e10', role: 'button' }, + ], + }, + }); + + expect(output.text).toBe( + [ + 'Tapped @e4 (100, 200)', + 'settled after 500ms: +0 -2 (~3 unchanged)', + '- @e4 [button] "OK"', + '- @e5 [text] "Are you sure?"', + 'unchanged interactive (2):', + '= @e9 [button] "Add to cart"', + '= @e10 [button]', + ].join('\n'), + ); + }); + + test('marks the unchanged interactive tail as truncated', () => { + const output = formatPress({ + message: 'Tapped @e4 (100, 200)', + x: 100, + y: 200, + settle: { + settled: true, + waitedMs: 500, + diff: { + summary: { additions: 0, removals: 1, unchanged: 20 }, + lines: [{ kind: 'removed', text: '@e4 [button] "OK"' }], + }, + tail: [{ ref: 'e9', role: 'button', label: 'Add to cart' }], + tailTruncated: true, + }, + }); + + expect(output.text).toBe( + [ + 'Tapped @e4 (100, 200)', + 'settled after 500ms: +0 -1 (~20 unchanged)', + '- @e4 [button] "OK"', + 'unchanged interactive (1):', + '= @e9 [button] "Add to cart"', + '… more interactive elements not shown, use snapshot -i', + ].join('\n'), + ); + }); + test('prints not-settled verdict without a dangling diff summary', () => { const output = formatPress({ message: 'Tapped (278, 817)', diff --git a/src/commands/interaction/output.ts b/src/commands/interaction/output.ts index 74217c5418..8324f9e746 100644 --- a/src/commands/interaction/output.ts +++ b/src/commands/interaction/output.ts @@ -62,6 +62,8 @@ type SettleTextView = { lines?: Array<{ kind?: string; text?: string }>; truncated?: boolean; }; + tail?: Array<{ ref?: string; role?: string; label?: string }>; + tailTruncated?: boolean; }; /** @@ -72,7 +74,11 @@ type SettleTextView = { function formatSettleText(settle: unknown): string { if (!settle || typeof settle !== 'object') return ''; const view = settle as SettleTextView; - const parts = [formatSettleVerdict(view), ...formatSettleDiffLines(view.diff)]; + const parts = [ + formatSettleVerdict(view), + ...formatSettleDiffLines(view.diff), + ...formatSettleTailLines(view), + ]; if (view.hint) parts.push(`hint: ${view.hint}`); return `\n${parts.join('\n')}`; } @@ -85,6 +91,23 @@ function formatSettleDiffLines(diff: SettleTextView['diff']): string[] { return lines; } +// Unchanged interactive tail: only present when the diff's added lines +// carried zero refs (modal-dismiss/toast-only diff), so the settled tree's +// remaining actionable elements would otherwise be invisible. +function formatSettleTailLines(view: SettleTextView): string[] { + const tail = view.tail ?? []; + if (tail.length === 0) return []; + const lines = [`unchanged interactive (${tail.length}):`]; + for (const entry of tail) { + const label = entry.label ? ` "${entry.label}"` : ''; + lines.push(`= @${entry.ref ?? ''} [${entry.role ?? ''}]${label}`); + } + if (view.tailTruncated) { + lines.push('… more interactive elements not shown, use snapshot -i'); + } + return lines; +} + function formatSettleVerdict(view: SettleTextView): string { const verdict = view.settled === true ? 'settled' : 'not settled'; const summary = view.diff?.summary; diff --git a/src/commands/interaction/runtime/settle.test.ts b/src/commands/interaction/runtime/settle.test.ts index 2cdc6c2235..e9ab3ab959 100644 --- a/src/commands/interaction/runtime/settle.test.ts +++ b/src/commands/interaction/runtime/settle.test.ts @@ -10,7 +10,7 @@ import { } from '../../../runtime.ts'; import { makeSnapshotState } from '../../../__tests__/test-utils/index.ts'; import { ref, selector } from './selector-read.ts'; -import { NEVER_SETTLED_HINT } from './settle.ts'; +import { buildSettleTailEntries, NEVER_SETTLED_HINT } from './settle.ts'; // #1101 --settle: quiet-window settle loop composition on the interaction // commands. Budgets are injected (fake clock) — no real waiting. @@ -120,6 +120,9 @@ test('press --settle returns the settled diff and stores the settled tree', asyn // The settled tree became the session snapshot: the diff's refs resolve. const stored = (await device.sessions.get('default')) as { snapshot?: SnapshotState }; assert.equal(stored.snapshot?.nodes[0]?.label, 'Welcome!'); + // Added lines already hand back a fresh target: the tail would be pure cost. + assert.equal(settle.tail, undefined); + assert.equal(settle.tailTruncated, undefined); }); test('never-settling content returns settled: false without an actionable diff', async () => { @@ -418,3 +421,239 @@ test('added lines win diff-budget slots over removals under truncation', async ( assert.equal(added.length, 10); assert.ok(added.every((line) => line.ref !== undefined)); }); + +// Unchanged interactive refs tail: benchmarks showed a removals-only settled +// diff (modal dismiss, toast dismiss) leaves the next actionable target +// invisible — the diff has nothing added to hand back. The tail fills that +// gap from the settled tree itself. + +function modalBeforeSnapshot(): SnapshotState { + return makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Button', + label: 'Add to cart', + rect: { x: 10, y: 20, width: 100, height: 40 }, + hittable: true, + }, + { + index: 1, + depth: 0, + type: 'StaticText', + label: 'Price: $12', + rect: { x: 10, y: 80, width: 100, height: 20 }, + }, + { + index: 2, + depth: 0, + type: 'Button', + label: 'OK', + rect: { x: 10, y: 140, width: 100, height: 40 }, + hittable: true, + }, + { + index: 3, + depth: 0, + type: 'Button', + label: 'Cancel', + rect: { x: 10, y: 200, width: 100, height: 40 }, + hittable: true, + }, + { + index: 4, + depth: 0, + type: 'StaticText', + label: 'Are you sure?', + rect: { x: 10, y: 260, width: 100, height: 20 }, + }, + ]); +} + +// Modal dismissed: only the background elements survive, and every one of +// them matches a `modalBeforeSnapshot` line exactly (unchanged), so the diff +// carries zero additions. +function modalDismissedSnapshot(): SnapshotState { + return makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Button', + label: 'Add to cart', + rect: { x: 10, y: 20, width: 100, height: 40 }, + hittable: true, + }, + { + index: 1, + depth: 0, + type: 'StaticText', + label: 'Price: $12', + rect: { x: 10, y: 80, width: 100, height: 20 }, + }, + ]); +} + +test('a removals-only settled diff (modal dismiss) attaches an unchanged interactive tail', async () => { + const before = modalBeforeSnapshot(); + const after = modalDismissedSnapshot(); + let captures = 0; + const device = createSettleDevice({ + stored: before, + captureSnapshot: () => { + captures += 1; + return { snapshot: captures === 1 ? before : after }; + }, + }); + + const result = await device.interactions.press(selector('label=OK'), { + session: 'default', + settle: {}, + }); + + const settle = result.settle; + assert.ok(settle); + assert.deepEqual(settle.diff?.summary, { additions: 0, removals: 3, unchanged: 2 }); + assert.ok(!settle.diff?.lines.some((line) => line.kind === 'added')); + // The StaticText survives but is not interactive; only the hittable button + // makes the tail. + assert.deepEqual(settle.tail, [{ ref: 'e1', role: 'button', label: 'Add to cart' }]); + assert.equal(settle.tailTruncated, undefined); +}); + +test('the unchanged interactive tail excludes non-hittable and covered candidates', async () => { + // Every surviving node's attributes (hittable/blocked-relevant fields) must + // match a `before` line exactly, or it would read as an added line and + // suppress the tail entirely (see the trigger-condition test above). + const survivors = [ + { + index: 0, + depth: 0, + type: 'Button', + label: 'Add to cart', + rect: { x: 10, y: 20, width: 100, height: 40 }, + hittable: false, + }, + { + index: 1, + depth: 0, + type: 'Button', + label: 'Wishlist', + rect: { x: 10, y: 80, width: 100, height: 40 }, + hittable: true, + }, + { + index: 2, + depth: 0, + type: 'Button', + label: 'Share', + rect: { x: 10, y: 140, width: 100, height: 40 }, + hittable: true, + }, + ]; + const before = makeSnapshotState([ + ...survivors, + { + index: 3, + depth: 0, + type: 'Button', + label: 'OK', + rect: { x: 10, y: 200, width: 100, height: 40 }, + hittable: true, + }, + { + index: 4, + depth: 0, + type: 'Button', + label: 'Cancel', + rect: { x: 10, y: 260, width: 100, height: 40 }, + hittable: true, + }, + ]); + // Same attributes as `survivors`; `interactionBlocked` is not part of the + // comparable key, so marking Wishlist covered here does not flip it to + // "added". + const after = makeSnapshotState([ + survivors[0]!, + { ...survivors[1]!, interactionBlocked: 'covered' as const }, + survivors[2]!, + ]); + let captures = 0; + const device = createSettleDevice({ + stored: before, + captureSnapshot: () => { + captures += 1; + return { snapshot: captures === 1 ? before : after }; + }, + }); + + const result = await device.interactions.press(selector('label=OK'), { + session: 'default', + settle: {}, + }); + + const settle = result.settle; + assert.ok(settle); + assert.equal(settle.diff?.summary.additions, 0); + // Not hittable, then covered, are both excluded; only the plain hittable + // button remains. + assert.deepEqual(settle.tail, [{ ref: 'e3', role: 'button', label: 'Share' }]); +}); + +test('the unchanged interactive tail is capped with a truncation marker', async () => { + const buttons = (labelPrefix: string, count: number, offset = 0) => + Array.from({ length: count }, (_, index) => ({ + index: index + offset, + depth: 0, + type: 'Button', + label: `${labelPrefix} ${index}`, + rect: { x: 0, y: index * 40, width: 100, height: 40 }, + hittable: true, + })); + // 25 surviving buttons plus 2 modal-only buttons that get removed. + const before = makeSnapshotState([...buttons('Row', 25), ...buttons('Modal', 2, 25)]); + const after = makeSnapshotState(buttons('Row', 25)); + let captures = 0; + const device = createSettleDevice({ + stored: before, + captureSnapshot: () => { + captures += 1; + return { snapshot: captures === 1 ? before : after }; + }, + }); + + const result = await device.interactions.press(selector('label="Modal 0"'), { + session: 'default', + settle: {}, + }); + + const settle = result.settle; + assert.ok(settle); + assert.equal(settle.diff?.summary.additions, 0); + assert.equal(settle.tail?.length, 20); + assert.equal(settle.tailTruncated, true); +}); + +test('buildSettleTailEntries dedups candidates already carrying an excluded ref', () => { + const settledNodes = makeSnapshotState([ + { + index: 0, + depth: 0, + type: 'Button', + label: 'Add to cart', + rect: { x: 10, y: 20, width: 100, height: 40 }, + hittable: true, + }, + { + index: 1, + depth: 0, + type: 'Button', + label: 'Share', + rect: { x: 10, y: 80, width: 100, height: 40 }, + hittable: true, + }, + ]).nodes; + + const result = buildSettleTailEntries(settledNodes, new Set(['e1'])); + + assert.deepEqual(result.tail, [{ ref: 'e2', role: 'button', label: 'Share' }]); +}); diff --git a/src/commands/interaction/runtime/settle.ts b/src/commands/interaction/runtime/settle.ts index b113ce8a61..9193d536c4 100644 --- a/src/commands/interaction/runtime/settle.ts +++ b/src/commands/interaction/runtime/settle.ts @@ -2,12 +2,14 @@ import type { SnapshotNode } from '../../../kernel/snapshot.ts'; import type { AgentDeviceRuntime, CommandContext } from '../../../runtime-contract.ts'; import { isSparseSnapshotQualityVerdict } from '../../../snapshot/snapshot-quality.ts'; import { buildSnapshotDiff } from '../../../snapshot/snapshot-diff.ts'; +import { displayLabel, formatRole } from '../../../snapshot/snapshot-lines.ts'; import { summarizeAxEvidence } from '../../../utils/ax-digest.ts'; import type { InteractionEvidence, ResolvedInteractionTarget, SettleObservation, SettleParams, + SettleTailEntry, } from '../../../contracts/interaction.ts'; import type { CapturedSnapshot } from './selector-read-shared.ts'; import { @@ -40,6 +42,11 @@ export type SettleOutcome = { // else. The summary always carries the true counts. const MAX_SETTLE_DIFF_LINES = 80; +// Unchanged-interactive-tail bound: same token-budget principle as the diff +// line cap, sized smaller since the tail is a fallback list, not the primary +// payload. +const MAX_SETTLE_TAIL_ENTRIES = 20; + export const NEVER_SETTLED_HINT = 'The UI kept changing for the whole settle budget (animation, carousel, or ticker?), so no settled diff is shown. Raise --timeout, wait for specific content, or take a fresh snapshot.'; @@ -87,7 +94,7 @@ export async function settleAfterInteraction( // observation, so surfacing refs would invite agents to act on // advisory state. ...(outcome.settled && stored - ? { diff: buildSettleDiff(resolveBaselineNodes(params.resolved), settledNodes) } + ? buildSettleDiffAndTail(resolveBaselineNodes(params.resolved), settledNodes) : {}), ...resolveSettleHint(outcome, stored, settledNodes.length), }, @@ -152,6 +159,61 @@ function buildSettleDiff( }; } +function buildSettleDiffAndTail( + baselineNodes: SnapshotNode[], + settledNodes: SnapshotNode[], +): Pick { + const diff = buildSettleDiff(baselineNodes, settledNodes); + return { diff, ...buildSettleTail(diff, settledNodes) }; +} + +/** + * Unchanged interactive refs tail: attached ONLY when the settled diff carries + * zero added-line refs (a modal-dismiss/toast-only diff shows removals but + * nothing added, so the next actionable target is otherwise invisible). Every + * hittable, uncovered element on the settled tree is a candidate; refs already + * present on the diff's added lines are excluded so the tail never repeats + * what the diff already handed the caller. + */ +export function buildSettleTail( + diff: NonNullable, + settledNodes: SnapshotNode[], +): Pick { + const addedRefs = new Set( + diff.lines.filter((line) => line.kind === 'added' && line.ref).map((line) => line.ref), + ); + if (addedRefs.size > 0) return {}; + return buildSettleTailEntries(settledNodes, addedRefs); +} + +/** + * The filtering/cap step behind `buildSettleTail`, split out so the dedup + * rule (excludeRefs) is unit-testable independent of the trigger condition + * above. + */ +export function buildSettleTailEntries( + settledNodes: SnapshotNode[], + excludeRefs: ReadonlySet, +): Pick { + const candidates = settledNodes.filter( + (node) => + node.ref && + node.hittable === true && + node.interactionBlocked !== 'covered' && + !excludeRefs.has(node.ref), + ); + if (candidates.length === 0) return {}; + const tail: SettleTailEntry[] = candidates.slice(0, MAX_SETTLE_TAIL_ENTRIES).map((node) => { + const role = formatRole(node.type ?? 'Element'); + const label = displayLabel(node, role); + return { ref: node.ref, role, ...(label ? { label } : {}) }; + }); + return { + tail, + ...(candidates.length > tail.length ? { tailTruncated: true as const } : {}), + }; +} + // The iOS QWERTY keyboard is ~50 Key nodes; a fill that summons it would spend // most of the capped line budget spelling out the keyboard instead of the // content change the agent actually asked to observe. The Keyboard container diff --git a/src/contracts/interaction.ts b/src/contracts/interaction.ts index 9e91b6d8fa..8f8dfeef01 100644 --- a/src/contracts/interaction.ts +++ b/src/contracts/interaction.ts @@ -89,6 +89,16 @@ export type SettleDiffLine = { ref?: string; }; +/** + * One still-present, actionable element on the settled tree, surfaced by the + * unchanged-interactive tail (see `SettleObservation.tail`). + */ +export type SettleTailEntry = { + ref: string; + role: string; + label?: string; +}; + /** * Opt-in (`--settle`, #1101) post-action settled observation for mutating * interaction commands. After the action, the daemon re-captures the @@ -138,6 +148,22 @@ export type SettleObservation = { /** Present (true) when lines were capped to the response bound. */ truncated?: boolean; }; + /** + * Unchanged interactive refs tail: benchmarks (July 2026) showed 27% of + * `--settle` actions were followed by a fallback `snapshot -i` because a + * change-only diff omits refs for elements that did not change — after a + * modal dismiss the diff shows only removals, and the next button to press + * (already on screen, untouched) is absent from the response. `tail` lists + * the settled tree's remaining hittable, uncovered interactive elements so + * the response stays actionable without that extra round trip. Attached + * ONLY when `diff` carries zero added-line refs (the modal-dismiss/ + * toast-only signature) — a diff with fresh added refs already hands the + * next target, so the tail would be pure byte cost. Refs already present on + * `diff`'s added lines are excluded. Capped; `tailTruncated` marks when + * candidates exceeded the cap. + */ + tail?: SettleTailEntry[]; + tailTruncated?: true; hint?: string; }; diff --git a/src/daemon/handlers/__tests__/interaction-settle.test.ts b/src/daemon/handlers/__tests__/interaction-settle.test.ts index b7f22afd21..e82984f04e 100644 --- a/src/daemon/handlers/__tests__/interaction-settle.test.ts +++ b/src/daemon/handlers/__tests__/interaction-settle.test.ts @@ -131,6 +131,8 @@ type SettlePayload = { summary: { additions: number; removals: number; unchanged: number }; lines: Array<{ kind: string; text: string; ref?: string }>; }; + tail?: Array<{ ref: string; role: string; label?: string }>; + tailTruncated?: boolean; hint?: string; }; @@ -192,6 +194,72 @@ test('press --settle responds with the settled diff, refsGeneration, and clears expect(session.snapshot?.nodes.some((node) => node.label === 'Welcome!')).toBe(true); }); +const MODAL_BEFORE_NODES = [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'Button', + label: 'Continue', + rect: { x: 10, y: 20, width: 120, height: 44 }, + hittable: true, + }, + { + index: 2, + parentIndex: 0, + type: 'Button', + label: 'OK', + rect: { x: 10, y: 100, width: 120, height: 44 }, + hittable: true, + }, +]; + +// Modal dismissed: Continue survives unchanged, OK is gone — a removals-only +// diff with nothing added. +const MODAL_AFTER_NODES = [ + { index: 0, type: 'Application', rect: { x: 0, y: 0, width: 390, height: 844 } }, + { + index: 1, + parentIndex: 0, + type: 'Button', + label: 'Continue', + rect: { x: 10, y: 20, width: 120, height: 44 }, + hittable: true, + }, +]; + +test('press --settle on a removals-only diff attaches the unchanged interactive tail at the diff generation', async () => { + const sessionStore = makeSessionStore(); + const sessionName = 'settle-tail'; + seedSession(sessionName, sessionStore); + mockCommandDispatch({ + snapshots: [MODAL_BEFORE_NODES, MODAL_AFTER_NODES, MODAL_AFTER_NODES], + }); + + const response = await handleInteractionCommands({ + req: { + token: 't', + session: sessionName, + command: 'press', + positionals: ['label=OK'], + flags: { ...SETTLE_FLAGS }, + }, + sessionName, + sessionStore, + contextFromFlags, + }); + + const data = expectOkData(response); + const settle = data.settle as SettlePayload; + expect(settle.diff?.summary).toEqual({ additions: 0, removals: 1, unchanged: 2 }); + expect(settle.diff?.lines.some((line) => line.kind === 'added')).toBe(false); + expect(settle.tail).toEqual([{ ref: 'e2', role: 'button', label: 'Continue' }]); + // Same generation as the diff: the tail rides the settled tree that was + // just stored as the session snapshot. + const session = sessionStore.get(sessionName) as SessionState; + expect(settle.refsGeneration).toBe(session.snapshotGeneration); +}); + test('press --settle keeps the stale-refs input warning while re-issuing fresh refs', async () => { const sessionStore = makeSessionStore(); const sessionName = 'settle-stale-ref'; diff --git a/src/daemon/response-views.ts b/src/daemon/response-views.ts index 81c1de9c6e..875265ecb9 100644 --- a/src/daemon/response-views.ts +++ b/src/daemon/response-views.ts @@ -146,14 +146,16 @@ function selectorReadView(data: DaemonResponseData, level: ResponseLevel): Daemo * responses stay byte-identical at every level. The digest keeps the verdict * fields and the changed-line COUNTS (`diff.summary`) plus `refsGeneration`, * and drops the diff line texts — the changed-count summary is the digest - * answer; the lines are the default-level payload. `full` returns today's - * shape unchanged (nothing richer is computed yet). + * answer; the lines are the default-level payload. The unchanged-interactive + * `tail` (when present) is capped to the same DIGEST_REF_LIMIT as the other + * ref lists here. `full` returns today's shape unchanged (nothing richer is + * computed yet). */ function interactionSettleView(data: DaemonResponseData, level: ResponseLevel): DaemonResponseData { if (level !== 'digest') return data; const settle = data.settle; if (!settle || typeof settle !== 'object' || Array.isArray(settle)) return data; - const { diff, ...rest } = settle as Record; + const { diff, tail, ...rest } = settle as Record; if (!diff || typeof diff !== 'object' || Array.isArray(diff)) return data; const diffRecord = diff as Record; const summary = diffRecord.summary; @@ -163,6 +165,7 @@ function interactionSettleView(data: DaemonResponseData, level: ResponseLevel): settle: { ...rest, ...(refs.length > 0 ? { refs } : {}), + ...(Array.isArray(tail) && tail.length > 0 ? { tail: tail.slice(0, DIGEST_REF_LIMIT) } : {}), diff: { summary }, }, }; diff --git a/src/mcp/__tests__/command-tools.test.ts b/src/mcp/__tests__/command-tools.test.ts index 865db22d40..12b1d5bd59 100644 --- a/src/mcp/__tests__/command-tools.test.ts +++ b/src/mcp/__tests__/command-tools.test.ts @@ -553,6 +553,42 @@ test('MCP merges digest-level settled refs too', async () => { }); }); +test('MCP merges per-ref pins from a settle response unchanged-interactive tail', async () => { + const runCalls: Array<{ name: string; input: unknown }> = []; + const executor = createCommandToolExecutor({ + createClient: () => ({}) as AgentDeviceClient, + runCommand: async (_client, name, input) => { + runCalls.push({ name, input }); + if (name === 'press') { + // A removals-only diff (modal dismiss) carries no added refs, so the + // tail is the only ref-issuing surface on this response. + return { + ref: 'e2', + settle: { + settled: true, + waitedMs: 60, + captures: 2, + quietMs: 25, + timeoutMs: 2000, + refsGeneration: 9, + diff: { + summary: { additions: 0, removals: 1, unchanged: 1 }, + lines: [{ kind: 'removed', text: '@e2 [button] "OK"' }], + }, + tail: [{ ref: 'e1', role: 'button', label: 'Continue' }], + }, + }; + } + return {}; + }, + }); + + await executor.execute('press', { session: 'demo', target: { kind: 'ref', ref: '@e2' } }); + await executor.execute('press', { session: 'demo', target: { kind: 'ref', ref: '@e1' } }); + + assert.deepEqual(runCalls[1]?.input, { session: 'demo', target: { kind: 'ref', ref: '@e1~s9' } }); +}); + test('MCP leaves pins untouched for plain (non-settle) interaction responses', async () => { const runCalls: Array<{ name: string; input: unknown }> = []; const executor = createCommandToolExecutor({ diff --git a/src/mcp/command-output-schemas.ts b/src/mcp/command-output-schemas.ts index d70e9d3b04..919bf69860 100644 --- a/src/mcp/command-output-schemas.ts +++ b/src/mcp/command-output-schemas.ts @@ -161,6 +161,20 @@ const settleObservationSchema: JsonSchema = objectSchema( ['summary', 'lines'], 'Settled diff vs the pre-action tree (changed lines only).', ), + tail: { + type: 'array', + description: + 'Unchanged interactive refs tail: still-present, actionable elements from the settled tree, attached only when diff carries zero added-line refs (a modal-dismiss/toast-only diff).', + items: objectSchema( + { + ref: stringSchema('Plain ref body (e12) minted from the stored settled tree.'), + role: stringSchema(), + label: stringSchema(), + }, + ['ref', 'role'], + ), + }, + tailTruncated: booleanSchema('Present (true) when tail candidates exceeded the response cap.'), hint: stringSchema(), }, ['settled', 'waitedMs', 'captures', 'quietMs', 'timeoutMs'], diff --git a/src/mcp/command-tools.ts b/src/mcp/command-tools.ts index e8e571f5ec..74c5eb9ca4 100644 --- a/src/mcp/command-tools.ts +++ b/src/mcp/command-tools.ts @@ -191,11 +191,11 @@ function mergeIssuedRefPins( /** * MERGE-ONLY, like the snapshot/find rule: refs on the settled diff's added - * lines move to the settle generation; every other pin stays put (the settle - * capture replaced the tree, so an old pin on an unchanged-looking element is - * exactly what makes the daemon warn precisely). No settle payload, no diff, - * no digest refs, or no generation → not an issuing response; pins are left - * untouched. + * lines (plus the unchanged-interactive `tail`, when present) move to the + * settle generation; every other pin stays put (the settle capture replaced + * the tree, so an old pin on an unchanged-looking element is exactly what + * makes the daemon warn precisely). No settle payload, no diff, no digest + * refs, or no generation → not an issuing response; pins are left untouched. */ function mergeSettleIssuedRefPins( refPinsByScope: Map>, @@ -210,6 +210,7 @@ function mergeSettleIssuedRefPins( const issuedRefs: string[] = []; collectRefBodies(lines, issuedRefs); collectRefBodies(settle.refs, issuedRefs); + collectRefBodies(settle.tail, issuedRefs); if (issuedRefs.length === 0) return; const pins = refPinsByScope.get(scopeKey) ?? new Map(); refPinsByScope.set(scopeKey, pins); diff --git a/test/integration/provider-scenarios/settle-observation.test.ts b/test/integration/provider-scenarios/settle-observation.test.ts index df6e3cbbeb..ed1649fcb9 100644 --- a/test/integration/provider-scenarios/settle-observation.test.ts +++ b/test/integration/provider-scenarios/settle-observation.test.ts @@ -60,6 +60,50 @@ const SETTLED_NODES = [ }, ]; +// A dialog dismiss: Continue survives unchanged underneath, Cancel is the +// dialog's own button (dismissed with it) — a removals-only diff. +const MODAL_BEFORE_NODES = [ + { + index: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 400, height: 800 }, + }, + { + index: 1, + parentIndex: 0, + type: 'Button', + label: 'Continue', + hittable: true, + rect: { x: 100, y: 300, width: 200, height: 44 }, + }, + { + index: 2, + parentIndex: 0, + type: 'Button', + label: 'Cancel', + hittable: true, + rect: { x: 100, y: 400, width: 200, height: 44 }, + }, +]; + +const MODAL_DISMISSED_NODES = [ + { + index: 0, + type: 'Application', + label: 'Example', + rect: { x: 0, y: 0, width: 400, height: 800 }, + }, + { + index: 1, + parentIndex: 0, + type: 'Button', + label: 'Continue', + hittable: true, + rect: { x: 100, y: 300, width: 200, height: 44 }, + }, +]; + function snapshotEntry(nodes: readonly unknown[]): ProviderScenarioProviderEntry { return { command: 'ios.runner.snapshot', @@ -137,6 +181,7 @@ test('Provider-backed integration press --settle returns the settled diff and fr summary: { additions: number; removals: number; unchanged: number }; lines: Array<{ kind: string; text: string; ref?: string }>; }; + tail?: Array<{ ref: string; role: string; label?: string }>; hint?: string; }; assert.ok(settle, 'press --settle must return a settle observation'); @@ -147,6 +192,9 @@ test('Provider-backed integration press --settle returns the settled diff and fr const added = settle.diff?.lines.find((line) => line.kind === 'added'); assert.match(added?.text ?? '', /Done/); assert.equal(added?.ref, 'e2'); + // The added line already hands back a fresh target: the diff's own refs + // are the actionable payload, so the tail stays off. + assert.equal(settle.tail, undefined); // The settled tree is never serialized into the response. assert.equal(pressData.nodes, undefined); @@ -237,3 +285,86 @@ test('Provider-backed integration never-settled press --settle does not issue di }, ); }); + +test('Provider-backed integration modal-dismiss press --settle attaches the unchanged interactive tail', async () => { + const runnerTranscript = createProviderTranscript([ + // snapshot -i: issues refs + snapshotEntry(MODAL_BEFORE_NODES), + // press label=Cancel --settle: resolution capture, tap, settle captures. + // The dialog closes leaving Continue in place — a removals-only diff with + // no added refs, so the tail is the only actionable-target payload. + snapshotEntry(MODAL_BEFORE_NODES), + tapEntry(200, 422), + snapshotEntry(MODAL_DISMISSED_NODES), + snapshotEntry(MODAL_DISMISSED_NODES), + // press @e2 (the Continue ref from the tail): tap on the stored tree. + tapEntry(200, 322), + ]); + const appleRunnerProvider = createAppleRunnerProviderFromTranscript( + runnerTranscript, + 'ios.runner', + ); + const appleTool = createRecordingAppleToolProvider({ + simctl: simctlListDevicesHandler('com.apple.CoreSimulator.SimRuntime.iOS-18-0', [ + { name: PROVIDER_SCENARIO_IOS_SIMULATOR.name, udid: DEVICE_ID }, + ]), + }); + + await withProviderScenarioResource( + async () => + await createProviderScenarioHarness({ + appleRunnerProvider: () => appleRunnerProvider, + appleToolProvider: () => appleTool.provider, + deviceInventoryProvider: async () => [PROVIDER_SCENARIO_IOS_SIMULATOR], + }), + async (daemon) => { + const open = await daemon.callCommand('open', [APP], { + platform: 'ios', + udid: DEVICE_ID, + }); + assertRpcOk(open); + + const snapshot = await daemon.callCommand('snapshot', [], { + snapshotInteractiveOnly: true, + }); + assertRpcOk(snapshot); + + const press = await daemon.callCommand('press', ['label=Cancel'], { + settle: true, + settleQuietMs: 25, + timeoutMs: 10_000, + }); + const pressData = assertRpcOk(press); + const settle = pressData.settle as { + settled: boolean; + refsGeneration?: number; + diff?: { + summary: { additions: number; removals: number; unchanged: number }; + lines: Array<{ kind: string; text: string; ref?: string }>; + }; + tail?: Array<{ ref: string; role: string; label?: string }>; + tailTruncated?: boolean; + }; + assert.ok(settle, 'press --settle must return a settle observation'); + assert.equal(settle.settled, true); + assert.deepEqual(settle.diff?.summary, { additions: 0, removals: 1, unchanged: 2 }); + assert.equal( + settle.diff?.lines.some((line) => line.kind === 'added'), + false, + ); + assert.deepEqual(settle.tail, [{ ref: 'e2', role: 'button', label: 'Continue' }]); + assert.equal(settle.tailTruncated, undefined); + assert.equal(typeof settle.refsGeneration, 'number'); + + // The tail's ref acts directly on the stored settled tree, same as an + // added-line ref would. + const followUp = await daemon.callCommand('press', ['@e2'], {}); + const followUpData = assertRpcOk(followUp); + assert.equal(followUpData.warning, undefined); + assert.equal(followUpData.x, 200); + assert.equal(followUpData.y, 322); + + runnerTranscript.assertComplete(); + }, + ); +}); From f886b6e3123465f5a2f8a36009d5abc2ed7847c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Thu, 9 Jul 2026 12:17:35 +0200 Subject: [PATCH 2/2] refactor: address fallow audit findings on the settle tail - drop the unused export on buildSettleTail (tests exercise the trigger through the public interaction path and the filter via buildSettleTailEntries) - extract the digest tail capping from interactionSettleView into a module-private helper to stay under the complexity gate --- src/commands/interaction/runtime/settle.ts | 2 +- src/daemon/response-views.ts | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/commands/interaction/runtime/settle.ts b/src/commands/interaction/runtime/settle.ts index 9193d536c4..06a6e7ed59 100644 --- a/src/commands/interaction/runtime/settle.ts +++ b/src/commands/interaction/runtime/settle.ts @@ -175,7 +175,7 @@ function buildSettleDiffAndTail( * present on the diff's added lines are excluded so the tail never repeats * what the diff already handed the caller. */ -export function buildSettleTail( +function buildSettleTail( diff: NonNullable, settledNodes: SnapshotNode[], ): Pick { diff --git a/src/daemon/response-views.ts b/src/daemon/response-views.ts index 875265ecb9..03e8e5538d 100644 --- a/src/daemon/response-views.ts +++ b/src/daemon/response-views.ts @@ -165,12 +165,17 @@ function interactionSettleView(data: DaemonResponseData, level: ResponseLevel): settle: { ...rest, ...(refs.length > 0 ? { refs } : {}), - ...(Array.isArray(tail) && tail.length > 0 ? { tail: tail.slice(0, DIGEST_REF_LIMIT) } : {}), + ...cappedSettleDigestTail(tail), diff: { summary }, }, }; } +function cappedSettleDigestTail(tail: unknown): Record { + if (!Array.isArray(tail) || tail.length === 0) return {}; + return { tail: tail.slice(0, DIGEST_REF_LIMIT) }; +} + type DigestRef = { ref: string }; function readSettleDigestRefs(lines: unknown): DigestRef[] {