diff --git a/src/__tests__/runtime-selector-read.test.ts b/src/__tests__/runtime-selector-read.test.ts index fe1fc76aea..8e50c318d3 100644 --- a/src/__tests__/runtime-selector-read.test.ts +++ b/src/__tests__/runtime-selector-read.test.ts @@ -185,6 +185,61 @@ test('runtime find get_text reads the matched node', async () => { assert.equal(result.node.label, 'Continue'); }); +test('runtime find wait reports sparse snapshot verdicts on the selector-read route', async () => { + const initialSnapshot = selectorSnapshot(); + const session = { name: 'default', snapshot: initialSnapshot }; + const sessions = { + get: () => session, + set: (record) => { + session.snapshot = record.snapshot ?? session.snapshot; + }, + } satisfies CommandSessionStore; + const sparseSnapshot = makeSnapshotState([ + { + index: 0, + type: 'Application', + }, + ]); + const device = createAgentDevice({ + backend: { + platform: 'ios', + captureSnapshot: async () => ({ + nodes: sparseSnapshot.nodes, + backend: 'xctest', + quality: { + state: 'sparse', + backend: 'private-ax', + reason: 'sparse tree', + reasonCode: 'sparse-tree', + }, + }), + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions, + policy: localCommandPolicy(), + clock: { + now: () => 0, + sleep: async () => {}, + }, + }); + + await assert.rejects( + () => + device.selectors.find({ + session: 'default', + locator: 'text', + query: 'Never appears', + action: 'wait', + timeoutMs: 100, + }), + (error: unknown) => + error instanceof Error && + error.message === 'find could not read the current accessibility tree' && + (error as { details?: { reason?: string } }).details?.reason === 'sparse tree', + ); + assert.equal(session.snapshot, initialSnapshot); +}); + test('runtime wait can use backend text search', async () => { const device = createSelectorDevice(selectorSnapshot(), { findText: true, diff --git a/src/commands/selector-read-shared.ts b/src/commands/selector-read-shared.ts index 76d3cd72bd..d60683e99e 100644 --- a/src/commands/selector-read-shared.ts +++ b/src/commands/selector-read-shared.ts @@ -6,6 +6,7 @@ import type { import { AppError } from '../utils/errors.ts'; import type { SnapshotNode, SnapshotState } from '../utils/snapshot.ts'; import { findNodeByRef, normalizeRef } from '../utils/snapshot.ts'; +import { isSparseSnapshotQualityVerdict } from '../utils/snapshot-quality.ts'; import { extractReadableText } from '../utils/text-surface.ts'; import { findNodeByLabel, now, toBackendContext } from './selector-read-utils.ts'; import type { SelectorSnapshotInput } from './command-input.ts'; @@ -55,9 +56,14 @@ export async function captureSelectorSnapshot( nodes: result.nodes ?? [], truncated: result.truncated, backend: result.backend as SnapshotState['backend'], + ...(result.quality ? { snapshotQuality: result.quality } : {}), createdAt: now(runtime), } satisfies SnapshotState); - if (captureOptions.updateSession && session) { + if ( + captureOptions.updateSession && + session && + !isSparseSnapshotQualityVerdict(snapshot.snapshotQuality) + ) { await runtime.sessions.set({ ...session, snapshot }); } return { sessionName, session, snapshot }; diff --git a/src/commands/selector-read.ts b/src/commands/selector-read.ts index ffa3e80b1b..3aba94b09f 100644 --- a/src/commands/selector-read.ts +++ b/src/commands/selector-read.ts @@ -2,6 +2,10 @@ import type { FindAction, FindLocator } from '../utils/finders.ts'; import { findBestMatchesByLocator } from '../utils/finders.ts'; import type { SnapshotNode } from '../utils/snapshot.ts'; import { findNodeByRef, normalizeRef } from '../utils/snapshot.ts'; +import { + isSparseSnapshotQualityVerdict, + type SnapshotQualityVerdict, +} from '../utils/snapshot-quality.ts'; import type { AgentDeviceRuntime, CommandContext } from '../runtime-contract.ts'; import { AppError } from '../utils/errors.ts'; import { @@ -406,12 +410,22 @@ async function findFirstLocatorMatch( updateSession: true, scope: shouldScopeFind(locator) ? options.query : undefined, }); + if (isSparseSnapshotQualityVerdict(capture.snapshot.snapshotQuality)) { + throw sparseSelectorSnapshotError(capture.snapshot.snapshotQuality); + } const match = findBestMatchesByLocator(capture.snapshot.nodes, locator, options.query, { requireRect: false, }).matches[0]; return { capture, match }; } +function sparseSelectorSnapshotError(verdict: SnapshotQualityVerdict): AppError { + return new AppError('COMMAND_FAILED', 'find could not read the current accessibility tree', { + reason: verdict.reason, + hint: 'The snapshot quality verdict is sparse. Use screenshot as visual truth, navigate with coordinates if needed, then retry find after reaching a readable screen.', + }); +} + async function waitForSelector( runtime: AgentDeviceRuntime, options: WaitCommandOptions, diff --git a/src/daemon/handlers/__tests__/find.test.ts b/src/daemon/handlers/__tests__/find.test.ts index 8f9773d22f..dae954767e 100644 --- a/src/daemon/handlers/__tests__/find.test.ts +++ b/src/daemon/handlers/__tests__/find.test.ts @@ -160,7 +160,141 @@ test('handleFindCommands click prefers on-screen duplicate text matches', async expect(invokeCalls[0]!.positionals?.[0]).toBe('@e3'); }); -test('handleFindCommands click retries full snapshot when iOS compact snapshot is sparse', async () => { +test('handleFindCommands click tries query-scoped full retry before failing sparse verdict', async () => { + const session = makeSession('default'); + session.snapshot = { + nodes: [ + { + index: 0, + ref: 'e1', + type: 'Application', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + ref: 'e2', + type: 'Button', + label: 'Previous Search', + rect: { x: 80, y: 792, width: 78, height: 48 }, + }, + ], + createdAt: Date.now(), + backend: 'xctest', + }; + mockDispatch.mockImplementation(async (_device, command) => { + if (command !== 'snapshot') return {}; + return { + backend: 'xctest', + quality: { + state: 'sparse', + backend: 'private-ax', + reason: 'sparse tree', + reasonCode: 'sparse-tree', + }, + nodes: [ + { + index: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 0, height: 0 }, + }, + ], + }; + }); + + const previousSnapshot = session.snapshot; + const { response, invokeCalls } = await runFindClickScenario({ + positionals: ['Search', 'click'], + session, + }); + + expect(response.ok).toBe(false); + expect(session.snapshot).toBe(previousSnapshot); + expect(invokeCalls).toHaveLength(0); + expect(!response.ok && response.error).toMatchObject({ + code: 'COMMAND_FAILED', + message: 'find could not read the current accessibility tree', + details: { + reason: 'sparse tree', + hint: expect.stringContaining('snapshot quality verdict is sparse'), + }, + }); + const snapshotCalls = mockDispatch.mock.calls.filter((call) => call[1] === 'snapshot'); + expect(snapshotCalls).toHaveLength(2); + expect(snapshotCalls[0]![4]).toMatchObject({ + snapshotInteractiveOnly: true, + snapshotCompact: true, + }); + expect(snapshotCalls[1]![4]).toMatchObject({ + snapshotInteractiveOnly: false, + snapshotCompact: false, + snapshotScope: 'Search', + }); +}); + +test('handleFindCommands click uses query-scoped full retry when sparse verdict recovers', async () => { + const snapshotResponses = [ + { + backend: 'xctest', + quality: { + state: 'sparse', + backend: 'private-ax', + reason: 'sparse tree', + reasonCode: 'sparse-tree', + }, + nodes: [ + { + index: 0, + type: 'Application', + rect: { x: 0, y: 0, width: 0, height: 0 }, + }, + ], + }, + { + backend: 'xctest', + quality: { + state: 'healthy', + backend: 'tree', + }, + nodes: [ + { + index: 0, + type: 'Application', + hittable: false, + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + type: 'Button', + label: 'Search', + hittable: true, + rect: { x: 80, y: 792, width: 78, height: 48 }, + parentIndex: 0, + }, + ], + }, + ]; + mockDispatch.mockImplementation(async (_device, command) => { + if (command !== 'snapshot') return {}; + return snapshotResponses.shift() ?? { nodes: [] }; + }); + + const { response, invokeCalls } = await runFindClickScenario({ + positionals: ['Search', 'click'], + }); + + expect(response.ok).toBe(true); + expect(invokeCalls[0]!.positionals?.[0]).toBe('@e1'); + expect(response.ok ? response.data : undefined).toMatchObject({ x: 119, y: 816 }); + const snapshotCalls = mockDispatch.mock.calls.filter((call) => call[1] === 'snapshot'); + expect(snapshotCalls).toHaveLength(2); + expect(snapshotCalls[1]![4]).toMatchObject({ + snapshotInteractiveOnly: false, + snapshotCompact: false, + snapshotScope: 'Search', + }); +}); + +test('handleFindCommands click retries full snapshot for legacy iOS sparse shape without verdict', async () => { const snapshotResponses = [ { backend: 'xctest', @@ -215,7 +349,7 @@ test('handleFindCommands click retries full snapshot when iOS compact snapshot i }); }); -test('handleFindCommands click scopes full retry when unscoped iOS fallback fails', async () => { +test('handleFindCommands click scopes full retry for legacy sparse shape when unscoped fallback fails', async () => { const snapshotResponses = [ { backend: 'xctest', @@ -507,6 +641,58 @@ test('handleFindCommands wait bypasses snapshot cache while Android freshness re expect(mockDispatch).toHaveBeenCalledTimes(2); }); +test('handleFindCommands wait reports sparse verdict through selector runtime route', async () => { + const session = makeSession('default'); + session.snapshot = { + nodes: [ + { + index: 0, + ref: 'e1', + type: 'Button', + label: 'Previous screen action', + rect: { x: 24, y: 600, width: 180, height: 52 }, + }, + ], + createdAt: Date.now(), + backend: 'xctest', + }; + const previousSnapshot = session.snapshot; + mockDispatch.mockImplementation(async (_device, command) => { + if (command !== 'snapshot') return {}; + return { + backend: 'xctest', + quality: { + state: 'sparse', + backend: 'private-ax', + reason: 'sparse tree', + reasonCode: 'sparse-tree', + }, + nodes: [ + { + index: 0, + type: 'Application', + }, + ], + }; + }); + + const { response } = await runFindClickScenario({ + positionals: ['text', 'Never appears', 'wait', '350'], + session, + }); + + expect(response.ok).toBe(false); + expect(session.snapshot).toBe(previousSnapshot); + expect(!response.ok && response.error).toMatchObject({ + code: 'COMMAND_FAILED', + message: 'find could not read the current accessibility tree', + details: { + reason: 'sparse tree', + hint: expect.stringContaining('snapshot quality verdict is sparse'), + }, + }); +}); + test('handleFindCommands wait captures fresh snapshots while polling', async () => { const { response } = await runFindClickScenario({ positionals: ['text', 'Never appears', 'wait', '350'], diff --git a/src/daemon/handlers/__tests__/react-native.test.ts b/src/daemon/handlers/__tests__/react-native.test.ts index 7128ac09e4..b6cf8233b3 100644 --- a/src/daemon/handlers/__tests__/react-native.test.ts +++ b/src/daemon/handlers/__tests__/react-native.test.ts @@ -638,6 +638,138 @@ test('react-native dismiss-overlay reports verified success after a clean post-d expect(response.data.nextCommand).toBeUndefined(); }); +test('react-native dismiss-overlay reports sparse verdict instead of no overlay detected', async () => { + const sessionName = 'rn-sparse-session'; + const sessionStore = makeSessionStore(); + const session = makeSession(sessionName); + session.snapshot = { + nodes: [ + { + index: 0, + ref: 'e1', + label: 'Previous screen action', + rect: { x: 24, y: 600, width: 180, height: 52 }, + }, + ], + createdAt: Date.now(), + }; + const previousSnapshot = session.snapshot; + sessionStore.set(sessionName, session); + mockCaptureSnapshot.mockResolvedValue({ + snapshot: { + nodes: [ + { + index: 0, + ref: 'e1', + type: 'Application', + }, + ], + createdAt: Date.now(), + snapshotQuality: { + state: 'sparse', + backend: 'private-ax', + reason: 'sparse tree', + reasonCode: 'sparse-tree', + }, + }, + }); + + const response = await handleReactNativeCommands({ + req: { + token: 't', + session: sessionName, + command: 'react-native', + positionals: ['dismiss-overlay'], + flags: {}, + }, + sessionName, + logPath: '/tmp/daemon.log', + sessionStore, + contextFromFlags: () => ({}), + }); + + expect(response?.ok).toBe(false); + expect(session.snapshot).toBe(previousSnapshot); + expect(mockDispatchCommand).not.toHaveBeenCalled(); + expect(!response?.ok && response?.error).toMatchObject({ + code: 'COMMAND_FAILED', + message: + 'React Native overlay state could not be determined because the accessibility tree is unreadable', + details: { + reason: 'sparse tree', + hint: expect.stringContaining('snapshot quality verdict is sparse'), + }, + }); +}); + +test('react-native dismiss-overlay reports unverified dismiss when post-dismiss snapshot is sparse', async () => { + const sessionName = 'rn-verify-sparse-session'; + const sessionStore = makeSessionStore(); + sessionStore.set(sessionName, makeSession(sessionName)); + mockDispatchCommand.mockResolvedValue({ x: 105, y: 714 }); + mockCaptureSnapshot + .mockResolvedValueOnce({ + snapshot: { + nodes: [ + { + index: 0, + ref: 'e1', + label: 'LogBox', + rect: { x: 0, y: 640, width: 390, height: 120 }, + }, + { + index: 1, + ref: 'e2', + label: 'Close', + rect: { x: 84, y: 692, width: 42, height: 44 }, + }, + ], + createdAt: Date.now(), + }, + }) + .mockResolvedValueOnce({ + snapshot: { + nodes: [ + { + index: 0, + ref: 'e1', + type: 'Application', + }, + ], + createdAt: Date.now(), + snapshotQuality: { + state: 'sparse', + backend: 'private-ax', + reason: 'sparse tree', + reasonCode: 'sparse-tree', + }, + }, + }); + + const response = await handleReactNativeCommands({ + req: { + token: 't', + session: sessionName, + command: 'react-native', + positionals: ['dismiss-overlay'], + flags: {}, + }, + sessionName, + logPath: '/tmp/daemon.log', + sessionStore, + contextFromFlags: () => ({}), + }); + + expect(response?.ok).toBe(true); + expect(response?.ok && response.data).toMatchObject({ + action: 'dismiss-overlay', + verified: false, + verificationRequired: true, + verificationWarning: expect.stringContaining('accessibility tree is unreadable'), + nextCommand: 'agent-device screenshot', + }); +}); + test('react-native dismiss-overlay reports still-visible overlays with recovery guidance', async () => { const sessionName = 'rn-verify-still-visible-session'; const sessionStore = makeSessionStore(); diff --git a/src/daemon/handlers/__tests__/snapshot-capture.test.ts b/src/daemon/handlers/__tests__/snapshot-capture.test.ts index 88362f6ca1..eb965f7d5f 100644 --- a/src/daemon/handlers/__tests__/snapshot-capture.test.ts +++ b/src/daemon/handlers/__tests__/snapshot-capture.test.ts @@ -14,6 +14,29 @@ test('buildSnapshotState handles completely empty data object', () => { expect(state.truncated).toBeUndefined(); }); +test('buildSnapshotState carries structured snapshot quality verdicts', () => { + const state = buildSnapshotState( + { + nodes: [{ index: 0, type: 'Application' }], + backend: 'xctest', + quality: { + state: 'sparse', + backend: 'private-ax', + reason: 'sparse tree', + reasonCode: 'sparse-tree', + }, + }, + { snapshotInteractiveOnly: true }, + ); + + expect(state.snapshotQuality).toMatchObject({ + state: 'sparse', + backend: 'private-ax', + reason: 'sparse tree', + reasonCode: 'sparse-tree', + }); +}); + test('buildSnapshotState handles nodes with missing fields', () => { const state = buildSnapshotState( { diff --git a/src/daemon/handlers/find.ts b/src/daemon/handlers/find.ts index 13092544ff..50a7025aef 100644 --- a/src/daemon/handlers/find.ts +++ b/src/daemon/handlers/find.ts @@ -19,6 +19,10 @@ import { errorResponse } from './response.ts'; import { getActiveAndroidSnapshotFreshness } from '../android-snapshot-freshness.ts'; import { stripInternalInteractionFlags } from '../interaction-outcome-policy.ts'; import { dispatchFindReadOnlyViaRuntime } from '../selector-runtime.ts'; +import { + isSparseSnapshotQualityVerdict, + type SnapshotQualityVerdict, +} from '../../utils/snapshot-quality.ts'; export { parseFindArgs } from '../../utils/finders.ts'; @@ -121,7 +125,11 @@ export async function handleFindCommands(params: { return handleFindWait(ctx, fetchNodes, locator, query, timeoutMs); } - const { nodes } = await fetchNodes(); + const snapshotResult = await fetchNodes(); + if (isSparseSnapshotQualityVerdict(snapshotResult.snapshotQuality)) { + return sparseFindSnapshotResponse(snapshotResult.snapshotQuality); + } + const { nodes } = snapshotResult; const matchResult = resolveFindMatch({ nodes, locator, @@ -150,7 +158,8 @@ export async function handleFindCommands(params: { return handler ? handler() : null; } -function isSparseIosInteractiveSnapshot(snapshot: SnapshotState): boolean { +function isLegacySparseIosInteractiveSnapshot(snapshot: SnapshotState): boolean { + if (snapshot.snapshotQuality) return false; if (snapshot.backend !== 'xctest' || snapshot.nodes.length !== 1) return false; return snapshot.nodes[0]?.type === 'Application'; } @@ -167,11 +176,14 @@ function findActionRequiresRect(action: string): boolean { return action === 'click' || action === 'focus' || action === 'fill' || action === 'type'; } -type FindNodeFetcher = () => Promise<{ +type FindSnapshotResult = { nodes: SnapshotState['nodes']; truncated?: boolean; backend?: SnapshotState['backend']; -}>; + snapshotQuality?: SnapshotQualityVerdict; +}; + +type FindNodeFetcher = () => Promise; function createFindNodeFetcher(params: { device: SessionState['device']; @@ -188,7 +200,7 @@ function createFindNodeFetcher(params: { const { device, session, req, logPath, locator, query, scope, interactiveOnly } = params; const { sessionStore, sessionName } = params; let lastSnapshotAt = 0; - let lastNodes: SnapshotState['nodes'] | null = null; + let lastSnapshotResult: FindSnapshotResult | null = null; const capture = async (snapshotScope: string | undefined, interactive: boolean) => { const { snapshot } = await captureSnapshot({ device, @@ -209,28 +221,56 @@ function createFindNodeFetcher(params: { // Re-use a snapshot captured within the last 750 ms to avoid redundant dumps during // rapid find iterations. Skipped when Android freshness tracking is active, because // the cached tree may already be stale from a recent navigation action. - if (lastNodes && now - lastSnapshotAt < 750 && !getActiveAndroidSnapshotFreshness(session)) { - return { nodes: lastNodes }; + if ( + lastSnapshotResult && + now - lastSnapshotAt < 750 && + !getActiveAndroidSnapshotFreshness(session) + ) { + return lastSnapshotResult; } let snapshot = await capture(scope, interactiveOnly); - if (interactiveOnly && isSparseIosInteractiveSnapshot(snapshot)) { + if (interactiveOnly && isLegacySparseIosInteractiveSnapshot(snapshot)) { snapshot = await recoverSparseInteractiveSnapshot({ capture, locator, query, scope }); + } else if ( + interactiveOnly && + isSparseSnapshotQualityVerdict(snapshot.snapshotQuality) && + shouldScopeFind(locator) + ) { + snapshot = await recoverSparseVerdictWithQueryScope({ capture, query, snapshot }); } - const nodes = snapshot.nodes; + const snapshotResult = { + nodes: snapshot.nodes, + truncated: snapshot.truncated, + backend: snapshot.backend, + snapshotQuality: snapshot.snapshotQuality, + }; lastSnapshotAt = now; - lastNodes = nodes; - if (session) { + lastSnapshotResult = snapshotResult; + if (session && !isSparseSnapshotQualityVerdict(snapshot.snapshotQuality)) { setSessionSnapshot(session, snapshot); sessionStore.set(sessionName, session); } - return { nodes, truncated: snapshot.truncated, backend: snapshot.backend }; + return snapshotResult; }; } +async function recoverSparseVerdictWithQueryScope(params: { + capture: (scope: string | undefined, interactive: boolean) => Promise; + query: string; + snapshot: SnapshotState; +}): Promise { + const { capture, query, snapshot } = params; + try { + return await capture(query, false); + } catch { + return snapshot; + } +} + /** - * A sparse compact-interactive iOS snapshot usually means the runner could not enumerate the - * tree, not that the screen is empty: retry with a full snapshot, and when even unscoped AX - * serialization fails on unrelated content, with a query-scoped full snapshot. + * Legacy iOS runners did not report a structured quality verdict. For those mixed-version + * sessions, the one-node application shape still means the runner could not enumerate the tree: + * retry with a full snapshot, then a query-scoped full snapshot if broad serialization fails. */ async function recoverSparseInteractiveSnapshot(params: { capture: (scope: string | undefined, interactive: boolean) => Promise; @@ -247,6 +287,13 @@ async function recoverSparseInteractiveSnapshot(params: { } } +function sparseFindSnapshotResponse(verdict: SnapshotQualityVerdict): DaemonResponse { + return errorResponse('COMMAND_FAILED', 'find could not read the current accessibility tree', { + reason: verdict.reason, + hint: 'The snapshot quality verdict is sparse. Use screenshot as visual truth, navigate with coordinates if needed, then retry find after reaching a readable screen.', + }); +} + function resolveFindMatch(params: { nodes: SnapshotState['nodes']; locator: FindLocator; @@ -391,7 +438,7 @@ function rectsMatch( async function handleFindWait( ctx: FindContext, - fetchNodes: () => Promise<{ nodes: SnapshotState['nodes'] }>, + fetchNodes: FindNodeFetcher, locator: FindLocator, query: string, timeoutMs: number | undefined, @@ -399,8 +446,14 @@ async function handleFindWait( const { req, sessionStore, session, command, publicFlags } = ctx; const timeout = timeoutMs ?? 10000; const start = Date.now(); + let sparseVerdict: SnapshotQualityVerdict | undefined; while (Date.now() - start < timeout) { - const { nodes } = await fetchNodes(); + const { nodes, snapshotQuality } = await fetchNodes(); + if (isSparseSnapshotQualityVerdict(snapshotQuality)) { + sparseVerdict = snapshotQuality; + await sleep(300); + continue; + } const match = findBestMatchesByLocator(nodes, locator, query, { requireRect: false }) .matches[0]; if (match) { @@ -416,6 +469,7 @@ async function handleFindWait( } await sleep(300); } + if (sparseVerdict) return sparseFindSnapshotResponse(sparseVerdict); return errorResponse('COMMAND_FAILED', 'find wait timed out'); } diff --git a/src/daemon/handlers/interaction-snapshot.ts b/src/daemon/handlers/interaction-snapshot.ts index 2675d20a82..96e1659034 100644 --- a/src/daemon/handlers/interaction-snapshot.ts +++ b/src/daemon/handlers/interaction-snapshot.ts @@ -5,6 +5,7 @@ import type { SnapshotState } from '../../utils/snapshot.ts'; import type { ContextFromFlags } from './interaction-common.ts'; import { captureSnapshot } from './snapshot-capture.ts'; import { setSessionSnapshot } from '../session-snapshot.ts'; +import { isSparseSnapshotQualityVerdict } from '../../utils/snapshot-quality.ts'; export type CaptureSnapshotForSession = ( session: SessionState, @@ -39,7 +40,9 @@ export async function captureSnapshotForSession( logPath: dispatchContext.logPath ?? '', androidFreshnessMode: options.androidFreshnessMode, }); - setSessionSnapshot(session, snapshot); - sessionStore.set(session.name, session); + if (!isSparseSnapshotQualityVerdict(snapshot.snapshotQuality)) { + setSessionSnapshot(session, snapshot); + sessionStore.set(session.name, session); + } return snapshot; } diff --git a/src/daemon/handlers/react-native.ts b/src/daemon/handlers/react-native.ts index 834c8a99e1..1a9c7251d6 100644 --- a/src/daemon/handlers/react-native.ts +++ b/src/daemon/handlers/react-native.ts @@ -9,6 +9,10 @@ import { normalizeError } from '../../utils/errors.ts'; import { stripUndefined } from '../../utils/parsing.ts'; import { successText } from '../../utils/success-text.ts'; import type { SnapshotState } from '../../utils/snapshot.ts'; +import { + isSparseSnapshotQualityVerdict, + type SnapshotQualityVerdict, +} from '../../utils/snapshot-quality.ts'; import type { DaemonResponse, SessionState } from '../types.ts'; import { errorResponse } from './response.ts'; import { captureSnapshotForSession } from './interaction-snapshot.ts'; @@ -40,6 +44,9 @@ export async function handleReactNativeCommands( params.contextFromFlags, { interactiveOnly: true }, ); + if (isSparseSnapshotQualityVerdict(snapshot.snapshotQuality)) { + return responseForSparseReactNativeOverlaySnapshot(snapshot.snapshotQuality); + } const overlay = analyzeReactNativeOverlay(snapshot.nodes); const target = overlay.primaryAction; if (!target) { @@ -84,6 +91,19 @@ function responseForMissingReactNativeOverlayTarget(overlayDetected: boolean): D ); } +function responseForSparseReactNativeOverlaySnapshot( + verdict: SnapshotQualityVerdict, +): DaemonResponse { + return errorResponse( + 'COMMAND_FAILED', + 'React Native overlay state could not be determined because the accessibility tree is unreadable', + { + reason: verdict.reason, + hint: 'The snapshot quality verdict is sparse. Use screenshot as visual truth; if an overlay is visible, report it or navigate with coordinates, then retry snapshot or dismiss-overlay on a readable screen.', + }, + ); +} + async function dismissReactNativeOverlayTarget( params: InteractionHandlerParams, session: SessionState, @@ -148,6 +168,14 @@ async function verifyReactNativeOverlayDismissal( params.contextFromFlags, { interactiveOnly: true }, ); + if (isSparseSnapshotQualityVerdict(verificationSnapshot.snapshotQuality)) { + return { + verified: false, + verificationWarning: + 'React Native overlay dismissal could not be verified because the post-dismiss accessibility tree is unreadable. Use screenshot as visual truth.', + nextCommand: 'agent-device screenshot', + }; + } const overlay = analyzeReactNativeOverlay(verificationSnapshot.nodes); if (!overlay.detected) { return { diff --git a/src/daemon/handlers/snapshot-capture.ts b/src/daemon/handlers/snapshot-capture.ts index cee245ccfc..6a1867365d 100644 --- a/src/daemon/handlers/snapshot-capture.ts +++ b/src/daemon/handlers/snapshot-capture.ts @@ -376,6 +376,7 @@ export function buildSnapshotState( nodes?: RawSnapshotNode[]; truncated?: boolean; backend?: SnapshotBackend; + quality?: unknown; }, flags: | (Pick< @@ -398,11 +399,13 @@ export function buildSnapshotState( const nodes = attachRefs( snapshotRaw ? presentableNodes : annotateCoveredSnapshotNodes(presentableNodes), ); + const snapshotQuality = snapshotCaptureAnnotationsFrom(data).quality; return { nodes, truncated: data?.truncated, createdAt: Date.now(), backend: data?.backend, + ...(snapshotQuality ? { snapshotQuality } : {}), presentationKey: buildSnapshotPresentationKey(snapshotPresentationOptionsFromFlags(flags)), // Only broad Android snapshots become freshness baselines. If the user asked for a scoped // or filtered view, preserve that output contract but avoid pretending it is safe for diff --git a/src/daemon/selector-runtime.ts b/src/daemon/selector-runtime.ts index 91dfabff5b..a1f2ffb5ec 100644 --- a/src/daemon/selector-runtime.ts +++ b/src/daemon/selector-runtime.ts @@ -36,6 +36,7 @@ import { import type { ContextFromFlags } from './handlers/interaction-common.ts'; import { setSessionSnapshot } from './session-snapshot.ts'; import { getActiveAndroidSnapshotFreshness } from './android-snapshot-freshness.ts'; +import { isSparseSnapshotQualityVerdict } from '../utils/snapshot-quality.ts'; import { describeAndroidEscapeSurface, detectAndroidEscapeSurface, @@ -590,7 +591,7 @@ function createSelectorBackend(params: { logPath: logPath ?? '', snapshotScope, }); - if (session) { + if (session && !isSparseSnapshotQualityVerdict(capture.snapshot.snapshotQuality)) { setSessionSnapshot(session, capture.snapshot); sessionStore.set(sessionName, session); } @@ -683,7 +684,7 @@ async function captureWaitSnapshot(params: { outPath: params.req.flags?.out, logPath: params.logPath ?? '', }); - if (params.session) { + if (params.session && !isSparseSnapshotQualityVerdict(capture.snapshot.snapshotQuality)) { setSessionSnapshot(params.session, capture.snapshot); params.sessionStore.set(params.sessionName, params.session); } diff --git a/src/utils/snapshot-quality.ts b/src/utils/snapshot-quality.ts index fd9404a14a..9da048c347 100644 --- a/src/utils/snapshot-quality.ts +++ b/src/utils/snapshot-quality.ts @@ -69,6 +69,12 @@ export function readSnapshotQualityVerdict(value: unknown): SnapshotQualityVerdi }; } +export function isSparseSnapshotQualityVerdict( + verdict: SnapshotQualityVerdict | undefined, +): verdict is SnapshotQualityVerdict { + return verdict?.state === 'sparse'; +} + /** Canonical warning lines for a verdict; the single place degradation is worded. */ export function renderSnapshotQualityWarnings( verdict: SnapshotQualityVerdict, diff --git a/src/utils/snapshot.ts b/src/utils/snapshot.ts index 5155d89387..430045e338 100644 --- a/src/utils/snapshot.ts +++ b/src/utils/snapshot.ts @@ -1,3 +1,5 @@ +import type { SnapshotQualityVerdict } from './snapshot-quality.ts'; + export type Rect = { x: number; y: number; @@ -69,6 +71,7 @@ export type SnapshotState = { createdAt: number; truncated?: boolean; backend?: SnapshotBackend; + snapshotQuality?: SnapshotQualityVerdict; comparisonSafe?: boolean; presentationKey?: string; };