From c4d2ac9363cf90af48ec48b032a36c413d5cef1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 8 Jul 2026 17:13:50 +0200 Subject: [PATCH 1/4] fix: handle Android IME overlays in snapshots --- .../__tests__/runtime-assertions.test.ts | 92 +++++++++++++++ .../__tests__/runtime-interactions.test.ts | 70 +++++++++++ .../maestro/runtime-android-overlays.ts | 62 ++++++++++ src/compat/maestro/runtime-assertions.ts | 60 +++++++++- src/compat/maestro/runtime-interactions.ts | 57 ++++++++- .../__tests__/snapshot-capture.test.ts | 92 +++++++++++++++ src/daemon/handlers/snapshot-capture.ts | 8 +- .../snapshot-content-recovery.test.ts | 110 ++++++++++++++++++ .../android/input-method-overlays.ts | 46 ++++++++ .../android/snapshot-content-recovery.ts | 18 +++ src/snapshot/snapshot-occlusion.ts | 68 +++++++++-- 11 files changed, 666 insertions(+), 17 deletions(-) create mode 100644 src/compat/maestro/runtime-android-overlays.ts create mode 100644 src/platforms/android/__tests__/snapshot-content-recovery.test.ts create mode 100644 src/platforms/android/input-method-overlays.ts diff --git a/src/compat/maestro/__tests__/runtime-assertions.test.ts b/src/compat/maestro/__tests__/runtime-assertions.test.ts index c83fd1f613..5d53fac5e6 100644 --- a/src/compat/maestro/__tests__/runtime-assertions.test.ts +++ b/src/compat/maestro/__tests__/runtime-assertions.test.ts @@ -555,6 +555,71 @@ test('invokeMaestroAssertVisible does not use Android raw fallback for generated ); }); +test('invokeMaestroAssertVisible dismisses Android Gboard handwriting tutorial before native wait', async () => { + const calls: Array<[string, string[] | undefined]> = []; + const snapshots = [ + gboardHandwritingTutorialSnapshot(), + snapshot([node('Input', { type: 'android.view.View' })]), + ]; + const response = await invokeMaestroAssertVisible({ + baseReq: { + token: 't', + session: 's', + flags: { platform: 'android' }, + }, + positionals: ['label="Input" || text="Input" || id="Input"', '60000'], + invoke: async (req): Promise => { + calls.push([req.command, req.positionals]); + if (req.command === 'snapshot') { + return { ok: true, data: snapshots.shift() ?? snapshot([node('Input')]) }; + } + if (req.command === 'click') return { ok: true, data: {} }; + if (req.command === 'wait') return { ok: true, data: { matches: 1 } }; + return { ok: false, error: { code: 'UNEXPECTED_COMMAND', message: req.command } }; + }, + }); + + assert.equal(response.ok, true); + assert.deepEqual(calls, [ + ['snapshot', []], + ['click', ['588', '2758']], + ['snapshot', []], + ]); +}); + +test('invokeMaestroAssertVisible bounds Android verification retries after native wait succeeds', async () => { + vi.useFakeTimers(); + + const calls: Array<[string, string[] | undefined]> = []; + const responsePromise = invokeMaestroAssertVisible({ + baseReq: { + token: 't', + session: 's', + flags: { platform: 'android' }, + }, + positionals: ['label="Input" || text="Input" || id="Input"', '60000'], + invoke: async (req): Promise => { + calls.push([req.command, req.positionals]); + if (req.command === 'snapshot') { + return { ok: true, data: snapshot([node('Loading')]) }; + } + if (req.command === 'wait') return { ok: true, data: { matches: 1 } }; + return { ok: false, error: { code: 'UNEXPECTED_COMMAND', message: req.command } }; + }, + }); + + await vi.advanceTimersByTimeAsync(6500); + const response = await responsePromise; + + assert.equal(response.ok, false); + assert.deepEqual(calls.slice(0, 3), [ + ['snapshot', []], + ['wait', ['Input', '60000']], + ['snapshot', []], + ]); + assert.ok(calls.filter(([command]) => command === 'snapshot').length < 40); +}); + test('invokeMaestroAssertVisible writes terminal snapshot artifacts for failed attempts', async () => { const artifactsDir = fs.mkdtempSync(path.join(os.tmpdir(), 'maestro-assert-artifacts-')); try { @@ -818,6 +883,33 @@ function node( }; } +function gboardHandwritingTutorialSnapshot(): SnapshotState { + return snapshot([ + node('Push Article', { + index: 2, + ref: 'e2', + type: 'android.widget.Button', + bundleId: 'org.reactnavigation.playground', + }), + node('Try out your stylus', { + index: 3, + ref: 'e3', + value: 'Try out your stylus', + bundleId: 'com.google.android.inputmethod.latin', + rect: { x: 376, y: 1650, width: 592, height: 90 }, + }), + node('Cancel', { + index: 4, + ref: 'e4', + value: 'Cancel', + identifier: 'android:id/closeButton', + type: 'android.widget.Button', + bundleId: 'com.google.android.inputmethod.latin', + rect: { x: 450, y: 2699, width: 276, height: 118 }, + }), + ]); +} + test('invokeMaestroAssertNotVisible accepts timeout overrides for short extended waits', async () => { vi.spyOn(Date, 'now').mockReturnValueOnce(0).mockReturnValueOnce(0).mockReturnValueOnce(300); diff --git a/src/compat/maestro/__tests__/runtime-interactions.test.ts b/src/compat/maestro/__tests__/runtime-interactions.test.ts index db499c7d21..a9dcf86fb2 100644 --- a/src/compat/maestro/__tests__/runtime-interactions.test.ts +++ b/src/compat/maestro/__tests__/runtime-interactions.test.ts @@ -211,6 +211,35 @@ test('invokeMaestroTapOn resolves visible Android non-interactive text from a re expect(clicks).toEqual([['248', '231']]); }); +test('invokeMaestroTapOn dismisses Android Gboard handwriting tutorial before app tap', async () => { + const snapshots = [gboardHandwritingTutorialSnapshot(), buttonSnapshot('Push Article')]; + const clicks: string[][] = []; + const response = await invokeMaestroTapOn({ + baseReq: { + token: 'test', + session: 'android-input', + flags: { platform: 'android' }, + }, + positionals: ['label="Push Article" || text="Push Article" || id="Push Article"'], + invoke: async (req: DaemonRequest): Promise => { + if (req.command === 'snapshot') { + return { ok: true, data: snapshots.shift() ?? buttonSnapshot('Push Article') }; + } + if (req.command === 'click') { + clicks.push(req.positionals ?? []); + return { ok: true, data: {} }; + } + return { ok: false, error: { code: 'UNEXPECTED_COMMAND', message: req.command } }; + }, + }); + + expect(response.ok).toBe(true); + expect(clicks).toEqual([ + ['588', '2758'], + ['201', '149'], + ]); +}); + test('invokeMaestroTapOn taps resolved iOS buttons by coordinates', async () => { const { response, clicks } = await runTapOn( 'label="Pop to top" || text="Pop to top" || id="Pop to top"', @@ -691,6 +720,47 @@ function overlayDismissButtonSnapshot(): SnapshotState { }; } +function gboardHandwritingTutorialSnapshot(): SnapshotState { + return { + createdAt: Date.now(), + nodes: [ + appNode(), + windowNode(), + { + index: 2, + ref: 'e3', + type: 'android.widget.Button', + label: 'Push Article', + bundleId: 'org.reactnavigation.playground', + depth: 21, + parentIndex: 1, + rect: { x: 142, y: 128.66666412353516, width: 118, height: 40 }, + }, + { + index: 3, + ref: 'e4', + type: 'android.widget.TextView', + label: 'Try out your stylus', + value: 'Try out your stylus', + bundleId: 'com.google.android.inputmethod.latin', + depth: 3, + rect: { x: 376, y: 1650, width: 592, height: 90 }, + }, + { + index: 4, + ref: 'e5', + type: 'android.widget.Button', + label: 'Cancel', + value: 'Cancel', + identifier: 'android:id/closeButton', + bundleId: 'com.google.android.inputmethod.latin', + depth: 4, + rect: { x: 450, y: 2699, width: 276, height: 118 }, + }, + ], + }; +} + function truncatedContentSnapshot(): SnapshotState { return { createdAt: Date.now(), diff --git a/src/compat/maestro/runtime-android-overlays.ts b/src/compat/maestro/runtime-android-overlays.ts new file mode 100644 index 0000000000..8ab401799e --- /dev/null +++ b/src/compat/maestro/runtime-android-overlays.ts @@ -0,0 +1,62 @@ +import type { Point, SnapshotNode, SnapshotState } from '../../kernel/snapshot.ts'; +import { + findAndroidGboardHandwritingTutorialCancel, + hasAndroidGboardHandwritingTutorial, + isAndroidInputMethodSnapshotNode, +} from '../../platforms/android/input-method-overlays.ts'; +import { emitDiagnostic } from '../../utils/diagnostics.ts'; +import type { MaestroRuntimeInvoke, ReplayBaseRequest } from './runtime-support.ts'; + +export async function dismissAndroidMaestroBlockingOverlay(params: { + baseReq: ReplayBaseRequest; + invoke: MaestroRuntimeInvoke; + snapshot: SnapshotState; + targetNode?: SnapshotNode; + selector: string; +}): Promise { + if (params.baseReq.flags?.platform !== 'android') return false; + if (isAndroidInputMethodSnapshotNode(params.targetNode)) return false; + + const cancel = findAndroidGboardHandwritingTutorialCancel(params.snapshot); + if (!cancel?.rect) return false; + + const point = centerPoint(cancel.rect); + emitDiagnostic({ + level: 'info', + phase: 'maestro_android_blocking_overlay_dismiss', + data: { + selector: params.selector, + overlay: 'gboard-handwriting-tutorial', + nodeIndex: cancel.index, + point, + }, + }); + + const response = await params.invoke({ + ...params.baseReq, + command: 'click', + positionals: [String(point.x), String(point.y)], + flags: { + ...params.baseReq.flags, + postGestureStabilization: true, + }, + }); + return response.ok; +} + +export function hasAndroidMaestroBlockingOverlay(params: { + baseReq: ReplayBaseRequest; + snapshot: SnapshotState; + targetNode?: SnapshotNode; +}): boolean { + if (params.baseReq.flags?.platform !== 'android') return false; + if (isAndroidInputMethodSnapshotNode(params.targetNode)) return false; + return hasAndroidGboardHandwritingTutorial(params.snapshot); +} + +function centerPoint(rect: NonNullable): Point { + return { + x: Math.round(rect.x + rect.width / 2), + y: Math.round(rect.y + rect.height / 2), + }; +} diff --git a/src/compat/maestro/runtime-assertions.ts b/src/compat/maestro/runtime-assertions.ts index 3348900bbe..e524e83899 100644 --- a/src/compat/maestro/runtime-assertions.ts +++ b/src/compat/maestro/runtime-assertions.ts @@ -8,6 +8,7 @@ import { emitDiagnostic } from '../../utils/diagnostics.ts'; import type { Point, SnapshotState } from '../../kernel/snapshot.ts'; import { buildSnapshotDisplayLines } from '../../snapshot/snapshot-lines.ts'; import { sleep } from '../../utils/timeouts.ts'; +import { dismissAndroidMaestroBlockingOverlay } from './runtime-android-overlays.ts'; import { pointForMaestroTapOnTarget } from './runtime-geometry.ts'; import { captureMaestroSnapshot, @@ -87,6 +88,13 @@ async function invokeNativeMaestroVisibleWaitWithSnapshotFallback( nativeWaitQuery: string, ): Promise { const nativeStartedAt = Date.now(); + const preflightResponse = await maybeDismissAndroidBlockingOverlayBeforeVisibleWait( + params, + args, + nativeStartedAt, + ); + if (preflightResponse) return preflightResponse; + const nativeResponse = await runNativeVisibleWait(params, args, nativeWaitQuery); if (nativeResponse.ok) { if (shouldVerifyNativeVisibleWait(params.baseReq)) { @@ -99,7 +107,7 @@ async function invokeNativeMaestroVisibleWaitWithSnapshotFallback( nativeStartedAt, ); if (failedSample.kind === 'return') return failedSample.response; - return await invokeSnapshotMaestroAssertVisible(params, args); + return await invokeSnapshotMaestroAssertVisible(params, visibleAssertionRetryArgs(args)); } } rememberMaestroVisibleContext(params.scope, args.selector); @@ -126,10 +134,45 @@ async function invokeNativeMaestroVisibleWaitWithSnapshotFallback( ); } +async function maybeDismissAndroidBlockingOverlayBeforeVisibleWait( + params: MaestroAssertionRuntimeParams, + args: MaestroVisibilityAssertionArgs, + startedAt: number, +): Promise { + if (!shouldPreflightAndroidBlockingOverlay(params.baseReq, args)) return null; + + const sample = await readMaestroVisibilitySample(params, args.selector, 'assertVisible'); + if (sample.visible) return visibleAssertionResponse(sample.response, args.selector, startedAt); + if (!sample.snapshot) return null; + + const dismissed = await dismissAndroidMaestroBlockingOverlay({ + baseReq: params.baseReq, + invoke: params.invoke, + snapshot: sample.snapshot, + selector: args.selector, + }); + if (!dismissed) return null; + + return await invokeSnapshotMaestroAssertVisible(params, { + ...args, + timeoutMs: visibleAssertionRetryTimeoutMs(args.timeoutMs), + }); +} + function shouldVerifyNativeVisibleWait(baseReq: ReplayBaseRequest): boolean { return baseReq.flags?.platform === 'android'; } +function shouldPreflightAndroidBlockingOverlay( + baseReq: ReplayBaseRequest, + args: MaestroVisibilityAssertionArgs, +): boolean { + // Long launch waits can otherwise spend the full native wait budget before + // snapshot verification notices environment-owned keyboard overlays. Ordinary + // waits rely on the normal Android verification snapshot and core occlusion. + return baseReq.flags?.platform === 'android' && args.timeoutMs >= 30_000; +} + async function runNativeVisibleWait( params: MaestroAssertionRuntimeParams, args: MaestroVisibilityAssertionArgs, @@ -277,7 +320,7 @@ async function confirmVisibleAfterAndroidRecovery( ): Promise { const retryArgs = { ...args, - timeoutMs: Math.min(args.timeoutMs, MAESTRO_ASSERTION_POLICY.assertVisibleRetryTimeoutMs), + timeoutMs: visibleAssertionRetryTimeoutMs(args.timeoutMs), }; const nativeWaitQuery = readNativeVisibleWaitQuery(params.baseReq, retryArgs.selector); if (!nativeWaitQuery) return await invokeSnapshotMaestroAssertVisible(params, retryArgs); @@ -431,6 +474,19 @@ function readVisibleAssertionDeadlineAction(params: { : 'finish'; } +function visibleAssertionRetryArgs( + args: MaestroVisibilityAssertionArgs, +): MaestroVisibilityAssertionArgs { + return { + ...args, + timeoutMs: visibleAssertionRetryTimeoutMs(args.timeoutMs), + }; +} + +function visibleAssertionRetryTimeoutMs(timeoutMs: number): number { + return Math.min(timeoutMs, MAESTRO_ASSERTION_POLICY.assertVisibleRetryTimeoutMs); +} + function isReactNativeOverlayBlockingAssertion(response: DaemonResponse): boolean { return ( !response.ok && diff --git a/src/compat/maestro/runtime-interactions.ts b/src/compat/maestro/runtime-interactions.ts index 2def284c73..134bf263f3 100644 --- a/src/compat/maestro/runtime-interactions.ts +++ b/src/compat/maestro/runtime-interactions.ts @@ -8,9 +8,14 @@ import { type ScrollDirection, } from '../../core/scroll-gesture.ts'; import type { ReplayVarScope } from '../../replay/vars.ts'; +import type { SnapshotState } from '../../kernel/snapshot.ts'; import { emitDiagnostic } from '../../utils/diagnostics.ts'; import { sleep } from '../../utils/timeouts.ts'; import { pointForMaestroTapOnTarget, swipeCoordinatesFromTarget } from './runtime-geometry.ts'; +import { + dismissAndroidMaestroBlockingOverlay, + hasAndroidMaestroBlockingOverlay, +} from './runtime-android-overlays.ts'; import { captureMaestroSnapshot, clearMaestroRecoverableInteraction, @@ -55,6 +60,10 @@ type MaestroTapOnParams = { scope?: ReplayVarScope; }; +type MaestroTapOnAttemptState = { + dismissedBlockingOverlay: boolean; +}; + type MaestroScreenSwipeResolution = | { ok: true; @@ -192,8 +201,9 @@ export async function invokeMaestroTapOn(params: MaestroTapOnParams): Promise { const fuzzyTextQuery = extractMaestroVisibleTextQuery(selector); - const attempt = await invokeMaestroResolvedTapOn(params, selector, options); + const attempt = await invokeMaestroResolvedTapOn(params, selector, options, attemptState); if (attempt.response.ok) return { retry: false, response: attempt.response }; if (attempt.targetResolved && fuzzyTextQuery) { return await invokeMaestroFuzzyTapOn(params, fuzzyTextQuery); @@ -488,11 +499,46 @@ async function invokeMaestroResolvedTapOn( params: MaestroTapOnParams, selector: string, options: MaestroTapOnOptions, + attemptState: MaestroTapOnAttemptState, ): Promise<{ response: DaemonResponse; targetResolved: boolean }> { const target = await resolveMaestroInteractionTarget(params, selector, options, 'tapOn', { promoteTapTarget: true, }); if (!target.ok) return { response: target.response, targetResolved: false }; + if ( + attemptState.dismissedBlockingOverlay && + hasAndroidMaestroBlockingOverlay({ + baseReq: params.baseReq, + snapshot: target.snapshot, + targetNode: target.target.node, + }) + ) { + return { + response: errorResponse( + 'COMMAND_FAILED', + 'Android input overlay is still blocking Maestro tap after dismissal attempt.', + { selector }, + ), + targetResolved: false, + }; + } + const dismissedOverlay = await dismissAndroidMaestroBlockingOverlay({ + baseReq: params.baseReq, + invoke: params.invoke, + snapshot: target.snapshot, + targetNode: target.target.node, + selector, + }); + if (dismissedOverlay) { + attemptState.dismissedBlockingOverlay = true; + return { + response: errorResponse( + 'COMMAND_FAILED', + 'Dismissed Android input overlay before Maestro tap; retrying target resolution.', + ), + targetResolved: false, + }; + } return await clickMaestroResolvedTarget(params, selector, target.target, options); } @@ -586,7 +632,8 @@ async function resolveMaestroInteractionTarget( commandLabel: string, resolutionOptions: { promoteTapTarget: boolean }, ): Promise< - { ok: true; target: ResolvedMaestroInteractionTarget } | { ok: false; response: DaemonResponse } + | { ok: true; target: ResolvedMaestroInteractionTarget; snapshot: SnapshotState } + | { ok: false; response: DaemonResponse } > { const snapshotResponse = await captureMaestroSnapshot({ ...params, raw: true }); return resolveMaestroInteractionTargetFromResponse( @@ -610,7 +657,7 @@ function resolveMaestroInteractionTargetFromResponse( resolutionOptions: { promoteTapTarget: boolean }, snapshotResponse: DaemonResponse, ): - | { ok: true; target: ResolvedMaestroInteractionTarget } + | { ok: true; target: ResolvedMaestroInteractionTarget; snapshot: SnapshotState } | { ok: false; response: DaemonResponse } { if (!snapshotResponse.ok) return { ok: false, response: snapshotResponse }; const snapshot = readSnapshotState(snapshotResponse.data); @@ -650,6 +697,7 @@ function resolveMaestroInteractionTargetFromResponse( rect: fuzzyResolution.rect, frame, }, + snapshot, }; } } @@ -671,6 +719,7 @@ function resolveMaestroInteractionTargetFromResponse( rect: resolution.rect, frame, }, + snapshot, }; } diff --git a/src/daemon/handlers/__tests__/snapshot-capture.test.ts b/src/daemon/handlers/__tests__/snapshot-capture.test.ts index 2f8a3ae0a9..f8aac6eb06 100644 --- a/src/daemon/handlers/__tests__/snapshot-capture.test.ts +++ b/src/daemon/handlers/__tests__/snapshot-capture.test.ts @@ -139,6 +139,98 @@ test('buildSnapshotState marks content covered by floating overlays as visible b expect(state.nodes.some((node) => node.type === 'TabBar')).toBe(true); }); +test('buildSnapshotState marks Android app content covered by IME overlays as blocked', () => { + const state = buildSnapshotState( + { + nodes: [ + { + index: 0, + depth: 0, + type: 'android.widget.FrameLayout', + bundleId: 'org.example', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'android.widget.Button', + label: 'Push Article', + bundleId: 'org.example', + rect: { x: 40, y: 600, width: 180, height: 56 }, + hittable: true, + }, + { + index: 2, + depth: 1, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + rect: { x: 0, y: 400, width: 390, height: 444 }, + }, + ], + backend: 'android', + }, + undefined, + ); + + expect(state.nodes.find((node) => node.label === 'Push Article')).toMatchObject({ + hittable: false, + interactionBlocked: 'covered', + presentationHints: ['covered'], + }); +}); + +test('buildSnapshotState treats large Android IME subtrees as one overlay root', () => { + const imeChildren = Array.from({ length: 2000 }, (_, offset) => ({ + index: offset + 3, + depth: 2, + parentIndex: 2, + type: 'android.widget.TextView', + label: `Keyboard suggestion ${offset}`, + bundleId: 'com.google.android.inputmethod.latin', + rect: { x: offset % 300, y: 500 + (offset % 200), width: 80, height: 32 }, + })); + + const state = buildSnapshotState( + { + nodes: [ + { + index: 0, + depth: 0, + type: 'android.widget.FrameLayout', + bundleId: 'org.example', + rect: { x: 0, y: 0, width: 390, height: 844 }, + }, + { + index: 1, + depth: 1, + parentIndex: 0, + type: 'android.widget.Button', + label: 'Covered action', + bundleId: 'org.example', + rect: { x: 40, y: 620, width: 180, height: 56 }, + hittable: true, + }, + { + index: 2, + depth: 1, + type: 'android.widget.FrameLayout', + bundleId: 'com.google.android.inputmethod.latin', + rect: { x: 0, y: 400, width: 390, height: 444 }, + }, + ...imeChildren, + ], + backend: 'android', + }, + undefined, + ); + + expect(state.nodes.find((node) => node.label === 'Covered action')).toMatchObject({ + hittable: false, + interactionBlocked: 'covered', + }); +}); + test('buildSnapshotState does not treat later generic hittable containers as covers', () => { const state = buildSnapshotState( { diff --git a/src/daemon/handlers/snapshot-capture.ts b/src/daemon/handlers/snapshot-capture.ts index 7cfff64ea3..346ad0b613 100644 --- a/src/daemon/handlers/snapshot-capture.ts +++ b/src/daemon/handlers/snapshot-capture.ts @@ -16,6 +16,7 @@ import { import { annotateCoveredSnapshotNodes } from '../../snapshot/snapshot-occlusion.ts'; import { normalizeSnapshotTree } from '../../snapshot/snapshot-tree.ts'; export { buildSnapshotVisibility } from '../../snapshot/snapshot-visibility.ts'; +import { isAndroidInputMethodSnapshotNode } from '../../platforms/android/input-method-overlays.ts'; import type { SessionState } from '../types.ts'; import { ANDROID_FRESHNESS_RETRY_DEADLINE_MS, @@ -402,7 +403,12 @@ export function buildSnapshotState( ? presentIosInteractiveSnapshot(scopedNodes) : scopedNodes; const nodes = attachRefs( - snapshotRaw ? presentableNodes : annotateCoveredSnapshotNodes(presentableNodes), + snapshotRaw + ? presentableNodes + : annotateCoveredSnapshotNodes(presentableNodes, { + isAdditionalOverlayNode: + data?.backend === 'android' ? isAndroidInputMethodSnapshotNode : undefined, + }), ); const snapshotQuality = snapshotCaptureAnnotationsFrom(data).quality; return { diff --git a/src/platforms/android/__tests__/snapshot-content-recovery.test.ts b/src/platforms/android/__tests__/snapshot-content-recovery.test.ts new file mode 100644 index 0000000000..468c3219dc --- /dev/null +++ b/src/platforms/android/__tests__/snapshot-content-recovery.test.ts @@ -0,0 +1,110 @@ +import { test, expect } from 'vitest'; +import { classifyAndroidHelperContentRecovery } from '../snapshot-content-recovery.ts'; + +test('keeps known IME blocking windows instead of falling back to covered app content', () => { + const decision = classifyAndroidHelperContentRecovery( + helperXml([ + node({ + windowType: 1, + packageName: 'org.reactnavigation.playground', + className: 'android.widget.FrameLayout', + }), + node({ + windowType: 2, + packageName: 'com.google.android.inputmethod.latin', + className: 'android.widget.FrameLayout', + }), + node({ + text: 'Try out your stylus', + packageName: 'com.google.android.inputmethod.latin', + className: 'android.widget.TextView', + }), + node({ + text: 'Cancel', + resourceId: 'android:id/closeButton', + packageName: 'com.google.android.inputmethod.latin', + className: 'android.widget.Button', + }), + ]), + { + backend: 'android-helper', + nodeCount: 4, + rootPresent: true, + windowCount: 2, + captureMode: 'interactive-windows', + }, + { foregroundAppPackage: 'org.reactnavigation.playground' }, + ); + + expect(decision).toBeUndefined(); +}); + +test('falls back when helper output has only one meaningful IME node', () => { + const decision = classifyAndroidHelperContentRecovery( + helperXml([ + node({ + windowType: 1, + packageName: 'org.reactnavigation.playground', + className: 'android.widget.FrameLayout', + }), + node({ + text: 'Try out your stylus', + packageName: 'com.google.android.inputmethod.latin', + className: 'android.widget.TextView', + }), + ]), + { + backend: 'android-helper', + nodeCount: 2, + rootPresent: true, + windowCount: 2, + captureMode: 'interactive-windows', + }, + { foregroundAppPackage: 'org.reactnavigation.playground' }, + ); + + expect(decision?.reason).toBe('content-poor-app-window'); + expect(decision?.diagnostics.helperInputMethodMeaningfulNodeCount).toBe(1); +}); + +test('falls back when helper output has no foreground app or IME content', () => { + const decision = classifyAndroidHelperContentRecovery( + helperXml([ + node({ + windowType: 1, + packageName: 'org.reactnavigation.playground', + className: 'android.widget.FrameLayout', + }), + node({ + text: 'Unrelated overlay', + packageName: 'com.example.overlay', + className: 'android.widget.TextView', + }), + ]), + { + backend: 'android-helper', + nodeCount: 2, + rootPresent: true, + windowCount: 1, + captureMode: 'interactive-windows', + }, + { foregroundAppPackage: 'org.reactnavigation.playground' }, + ); + + expect(decision?.reason).toBe('content-poor-app-window'); + expect(decision?.diagnostics.helperInputMethodMeaningfulNodeCount).toBe(0); +}); + +function helperXml(nodes: string[]): string { + return `${nodes.join('')}`; +} + +function node(options: { + text?: string; + resourceId?: string; + packageName: string; + className: string; + windowType?: number; +}): string { + return ``; +} diff --git a/src/platforms/android/input-method-overlays.ts b/src/platforms/android/input-method-overlays.ts new file mode 100644 index 0000000000..a482fb226a --- /dev/null +++ b/src/platforms/android/input-method-overlays.ts @@ -0,0 +1,46 @@ +import type { RawSnapshotNode, SnapshotNode, SnapshotState } from '../../kernel/snapshot.ts'; +import { classifyAndroidInputOwnership } from './input-ownership.ts'; + +const GBOARD_PACKAGE = 'com.google.android.inputmethod.latin'; +const GBOARD_HANDWRITING_TITLE = 'Try out your stylus'; +const GBOARD_HANDWRITING_CANCEL_ID = 'android:id/closeButton'; +const GBOARD_HANDWRITING_CANCEL_LABEL = 'Cancel'; + +export function isAndroidInputMethodSnapshotNode( + node: Pick | undefined, +): boolean { + if (!node) return false; + return classifyAndroidInputOwnership({ + packageName: node.bundleId, + resourceId: node.identifier, + }).inputMethodOwned; +} + +export function findAndroidGboardHandwritingTutorialCancel( + snapshot: SnapshotState, +): SnapshotNode | undefined { + if (!snapshot.nodes.some(isAndroidGboardHandwritingTutorialTitle)) return undefined; + return snapshot.nodes.find(isAndroidGboardHandwritingTutorialCancel); +} + +export function hasAndroidGboardHandwritingTutorial(snapshot: SnapshotState): boolean { + return snapshot.nodes.some(isAndroidGboardHandwritingTutorialTitle); +} + +function isAndroidGboardHandwritingTutorialTitle(node: SnapshotNode): boolean { + return isGboardSnapshotNode(node) && nodeTextValues(node).includes(GBOARD_HANDWRITING_TITLE); +} + +function isAndroidGboardHandwritingTutorialCancel(node: SnapshotNode): boolean { + if (!isGboardSnapshotNode(node)) return false; + if (node.identifier === GBOARD_HANDWRITING_CANCEL_ID) return true; + return nodeTextValues(node).includes(GBOARD_HANDWRITING_CANCEL_LABEL); +} + +function isGboardSnapshotNode(node: SnapshotNode): boolean { + return node.bundleId === GBOARD_PACKAGE && isAndroidInputMethodSnapshotNode(node); +} + +function nodeTextValues(node: SnapshotNode): string[] { + return [node.label, node.value].filter((value): value is string => Boolean(value)); +} diff --git a/src/platforms/android/snapshot-content-recovery.ts b/src/platforms/android/snapshot-content-recovery.ts index a9ae3417fc..9199b197dc 100644 --- a/src/platforms/android/snapshot-content-recovery.ts +++ b/src/platforms/android/snapshot-content-recovery.ts @@ -1,9 +1,11 @@ import type { AndroidSnapshotBackendMetadata } from './snapshot-types.ts'; +import { isAndroidInputMethodOwnedNode } from './input-ownership.ts'; import { androidUiNodes, type AndroidUiNodeMetadata } from './ui-hierarchy.ts'; const ANDROID_WINDOW_TYPE_APPLICATION = 1; const MAX_REPORTED_WINDOW_TYPES = 8; const MIN_FOREGROUND_APP_MEANINGFUL_NODES = 2; +const MIN_INPUT_METHOD_MEANINGFUL_NODES = 2; const ANDROID_SYSTEM_PACKAGES = new Set(['android', 'com.android.systemui']); export type AndroidHelperContentRecoveryDecision = { @@ -16,6 +18,7 @@ export type AndroidHelperContentRecoveryDecision = { helperMeaningfulNodeCount: number; helperApplicationMeaningfulNodeCount: number; helperNonSystemMeaningfulNodeCount: number; + helperInputMethodMeaningfulNodeCount: number; helperForegroundAppMeaningfulNodeCount?: number; helperForegroundAppPackage?: string; helperForegroundAppMeaningfulNodeThreshold?: number; @@ -48,6 +51,9 @@ export function classifyAndroidHelperContentRecovery( (foregroundAppMeaningfulNodeCount < MIN_FOREGROUND_APP_MEANINGFUL_NODES && summary.meaningfulNodeCount > foregroundAppMeaningfulNodeCount)) ) { + if (summary.inputMethodMeaningfulNodeCount >= MIN_INPUT_METHOD_MEANINGFUL_NODES) { + return undefined; + } return buildRecoveryDecision( summary, metadata, @@ -115,6 +121,7 @@ type AndroidHelperXmlSummary = { meaningfulNodeCount: number; applicationMeaningfulNodeCount: number; nonSystemMeaningfulNodeCount: number; + inputMethodMeaningfulNodeCount: number; foregroundAppPackage?: string; foregroundAppMeaningfulNodeCount?: number; windowTypes: number[]; @@ -148,6 +155,7 @@ function createAndroidHelperXmlSummaryState( meaningfulNodeCount: 0, applicationMeaningfulNodeCount: 0, nonSystemMeaningfulNodeCount: 0, + inputMethodMeaningfulNodeCount: 0, ...(foregroundAppPackage !== undefined ? { foregroundAppPackage, foregroundAppMeaningfulNodeCount: 0 } : {}), @@ -191,6 +199,14 @@ function recordAndroidHelperMeaningfulNode( if (!isAndroidSystemPackage(node.packageName)) { summary.nonSystemMeaningfulNodeCount += 1; } + if ( + isAndroidInputMethodOwnedNode({ + packageName: node.packageName, + resourceId: node.resourceId, + }) + ) { + summary.inputMethodMeaningfulNodeCount += 1; + } if ( summary.foregroundAppPackage !== undefined && node.packageName === summary.foregroundAppPackage @@ -209,6 +225,7 @@ function finalizeAndroidHelperXmlSummary( meaningfulNodeCount: summary.meaningfulNodeCount, applicationMeaningfulNodeCount: summary.applicationMeaningfulNodeCount, nonSystemMeaningfulNodeCount: summary.nonSystemMeaningfulNodeCount, + inputMethodMeaningfulNodeCount: summary.inputMethodMeaningfulNodeCount, ...(summary.foregroundAppPackage !== undefined ? { foregroundAppPackage: summary.foregroundAppPackage, @@ -243,6 +260,7 @@ function buildRecoveryDiagnostics( helperMeaningfulNodeCount: summary.meaningfulNodeCount, helperApplicationMeaningfulNodeCount: summary.applicationMeaningfulNodeCount, helperNonSystemMeaningfulNodeCount: summary.nonSystemMeaningfulNodeCount, + helperInputMethodMeaningfulNodeCount: summary.inputMethodMeaningfulNodeCount, ...(summary.foregroundAppPackage !== undefined ? { helperForegroundAppPackage: summary.foregroundAppPackage, diff --git a/src/snapshot/snapshot-occlusion.ts b/src/snapshot/snapshot-occlusion.ts index b097e93f18..3412ab8d2f 100644 --- a/src/snapshot/snapshot-occlusion.ts +++ b/src/snapshot/snapshot-occlusion.ts @@ -34,20 +34,32 @@ const SEMANTIC_TOUCH_KIND_FRAGMENTS = [ type OcclusionScan = { nodes: RawSnapshotNode[]; byIndex: Map; + overlayPositions: number[]; }; -export function annotateCoveredSnapshotNodes(nodes: RawSnapshotNode[]): RawSnapshotNode[] { +export type SnapshotOcclusionOptions = { + isAdditionalOverlayNode?: (node: RawSnapshotNode) => boolean; +}; + +export function annotateCoveredSnapshotNodes( + nodes: RawSnapshotNode[], + options: SnapshotOcclusionOptions = {}, +): RawSnapshotNode[] { if (nodes.length < 2) return nodes; const annotated = [...nodes]; + const byIndex = new Map(annotated.map((node) => [node.index, node])); const scan: OcclusionScan = { nodes: annotated, - byIndex: new Map(annotated.map((node) => [node.index, node])), + byIndex, + overlayPositions: annotated.flatMap((node, position) => + isOverlayLikeNode(node, byIndex, options) ? [position] : [], + ), }; let changed = false; for (const [position, node] of annotated.entries()) { if (!isCandidateTouchNode(node)) continue; - const cover = findCoveringNode(scan, position, node); + const cover = findCoveringNode(scan, position, node, options); if (!cover) continue; changed = true; const coveredNode = { @@ -73,14 +85,18 @@ function findCoveringNode( scan: OcclusionScan, targetPosition: number, target: RawSnapshotNode, + options: SnapshotOcclusionOptions, ): RawSnapshotNode | null { const targetRect = positiveRect(target.rect); if (!targetRect) return null; const center = centerOfRect(targetRect); - for (let position = targetPosition + 1; position < scan.nodes.length; position += 1) { + for (const position of scan.overlayPositions) { + if (position <= targetPosition) continue; const candidate = scan.nodes[position]; - if (candidate && canCoverPoint(scan, position, target, targetRect, center)) return candidate; + if (candidate && canCoverPoint(scan, position, target, targetRect, center, options)) { + return candidate; + } } return null; @@ -92,10 +108,11 @@ function canCoverPoint( target: RawSnapshotNode, targetRect: Rect, point: { x: number; y: number }, + options: SnapshotOcclusionOptions, ): boolean { const candidate = scan.nodes[candidatePosition]; if (!candidate) return false; - const coverRect = visibleCoverRect(scan, candidatePosition, target, targetRect); + const coverRect = visibleCoverRect(scan, candidatePosition, target, targetRect, options); return Boolean(coverRect && containsPoint(coverRect, point.x, point.y)); } @@ -104,13 +121,14 @@ function visibleCoverRect( candidatePosition: number, target: RawSnapshotNode, targetRect: Rect, + options: SnapshotOcclusionOptions, ): Rect | null { const candidate = scan.nodes[candidatePosition]; - if (!candidate || !isOverlayLikeNode(candidate)) return null; + if (!candidate || !isOverlayLikeNode(candidate, scan.byIndex, options)) return null; if (areRelatedSnapshotNodes(target, candidate, scan.byIndex)) return null; const candidateRect = positiveRect(candidate.rect); if (!candidateRect || areRectsApproximatelyEqual(targetRect, candidateRect)) return null; - if (findCoveringNode(scan, candidatePosition, candidate)) return null; + if (findCoveringNode(scan, candidatePosition, candidate, options)) return null; return candidateRect; } @@ -121,12 +139,42 @@ function isCandidateTouchNode(node: RawSnapshotNode): boolean { return Boolean(node.label?.trim() || node.value?.trim() || node.identifier?.trim()); } -function isOverlayLikeNode(node: RawSnapshotNode): boolean { +function isOverlayLikeNode( + node: RawSnapshotNode, + byIndex: Map, + options: SnapshotOcclusionOptions, +): boolean { if (!positiveRect(node.rect)) return false; if (isViewportRoot(node)) return false; // This is a presentation-order heuristic: only known floating UI chrome should cover // later targets. Generic hittable containers can appear later without being visually on top. - return nodeKindIncludesAny(node, OVERLAY_KIND_FRAGMENTS); + return ( + nodeKindIncludesAny(node, OVERLAY_KIND_FRAGMENTS) || + isAdditionalOverlayRootNode(node, byIndex, options) + ); +} + +function isAdditionalOverlayRootNode( + node: RawSnapshotNode, + byIndex: Map, + options: SnapshotOcclusionOptions, +): boolean { + if (options.isAdditionalOverlayNode?.(node) !== true) return false; + let current = typeof node.parentIndex === 'number' ? byIndex.get(node.parentIndex) : undefined; + const visited = new Set(); + while (current && !visited.has(current.index)) { + if ( + options.isAdditionalOverlayNode?.(current) === true && + positiveRect(current.rect) && + !isViewportRoot(current) + ) { + return false; + } + visited.add(current.index); + current = + typeof current.parentIndex === 'number' ? byIndex.get(current.parentIndex) : undefined; + } + return true; } function isSemanticTouchNode(node: RawSnapshotNode): boolean { From 8b5d456e3c43ae2cf6756c1146e742772e46d9fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 8 Jul 2026 17:31:24 +0200 Subject: [PATCH 2/4] fix: satisfy Android IME CI guards --- .../maestro/runtime-android-overlays.ts | 24 ++---- src/compat/maestro/runtime-click.ts | 26 ++++++ src/compat/maestro/runtime-interactions.ts | 21 +---- .../android-input-ownership.ts} | 0 src/daemon/handlers/snapshot-capture.ts | 2 +- .../android/__tests__/input-ownership.test.ts | 2 +- src/platforms/android/device-input-state.ts | 2 +- src/platforms/android/fill-verification.ts | 2 +- .../android/snapshot-content-recovery.ts | 85 +++++++++++++------ .../android-input-method-overlays.ts} | 4 +- src/snapshot/snapshot-occlusion.ts | 29 +++++-- 11 files changed, 120 insertions(+), 77 deletions(-) create mode 100644 src/compat/maestro/runtime-click.ts rename src/{platforms/android/input-ownership.ts => core/android-input-ownership.ts} (100%) rename src/{platforms/android/input-method-overlays.ts => snapshot/android-input-method-overlays.ts} (94%) diff --git a/src/compat/maestro/runtime-android-overlays.ts b/src/compat/maestro/runtime-android-overlays.ts index 8ab401799e..67c390aea8 100644 --- a/src/compat/maestro/runtime-android-overlays.ts +++ b/src/compat/maestro/runtime-android-overlays.ts @@ -1,10 +1,11 @@ -import type { Point, SnapshotNode, SnapshotState } from '../../kernel/snapshot.ts'; +import type { SnapshotNode, SnapshotState } from '../../kernel/snapshot.ts'; import { findAndroidGboardHandwritingTutorialCancel, hasAndroidGboardHandwritingTutorial, isAndroidInputMethodSnapshotNode, -} from '../../platforms/android/input-method-overlays.ts'; +} from '../../snapshot/android-input-method-overlays.ts'; import { emitDiagnostic } from '../../utils/diagnostics.ts'; +import { centerSnapshotRectPoint, invokeMaestroClickPoint } from './runtime-click.ts'; import type { MaestroRuntimeInvoke, ReplayBaseRequest } from './runtime-support.ts'; export async function dismissAndroidMaestroBlockingOverlay(params: { @@ -20,7 +21,7 @@ export async function dismissAndroidMaestroBlockingOverlay(params: { const cancel = findAndroidGboardHandwritingTutorialCancel(params.snapshot); if (!cancel?.rect) return false; - const point = centerPoint(cancel.rect); + const point = centerSnapshotRectPoint(cancel.rect); emitDiagnostic({ level: 'info', phase: 'maestro_android_blocking_overlay_dismiss', @@ -32,15 +33,7 @@ export async function dismissAndroidMaestroBlockingOverlay(params: { }, }); - const response = await params.invoke({ - ...params.baseReq, - command: 'click', - positionals: [String(point.x), String(point.y)], - flags: { - ...params.baseReq.flags, - postGestureStabilization: true, - }, - }); + const response = await invokeMaestroClickPoint({ ...params, point }); return response.ok; } @@ -53,10 +46,3 @@ export function hasAndroidMaestroBlockingOverlay(params: { if (isAndroidInputMethodSnapshotNode(params.targetNode)) return false; return hasAndroidGboardHandwritingTutorial(params.snapshot); } - -function centerPoint(rect: NonNullable): Point { - return { - x: Math.round(rect.x + rect.width / 2), - y: Math.round(rect.y + rect.height / 2), - }; -} diff --git a/src/compat/maestro/runtime-click.ts b/src/compat/maestro/runtime-click.ts new file mode 100644 index 0000000000..6eb58b1765 --- /dev/null +++ b/src/compat/maestro/runtime-click.ts @@ -0,0 +1,26 @@ +import type { DaemonResponse } from '../../daemon/types.ts'; +import type { Point, SnapshotNode } from '../../kernel/snapshot.ts'; +import type { MaestroRuntimeInvoke, ReplayBaseRequest } from './runtime-support.ts'; + +export async function invokeMaestroClickPoint(params: { + baseReq: ReplayBaseRequest; + invoke: MaestroRuntimeInvoke; + point: Point; +}): Promise { + return await params.invoke({ + ...params.baseReq, + command: 'click', + positionals: [String(params.point.x), String(params.point.y)], + flags: { + ...params.baseReq.flags, + postGestureStabilization: true, + }, + }); +} + +export function centerSnapshotRectPoint(rect: NonNullable): Point { + return { + x: Math.round(rect.x + rect.width / 2), + y: Math.round(rect.y + rect.height / 2), + }; +} diff --git a/src/compat/maestro/runtime-interactions.ts b/src/compat/maestro/runtime-interactions.ts index 134bf263f3..ebcb536251 100644 --- a/src/compat/maestro/runtime-interactions.ts +++ b/src/compat/maestro/runtime-interactions.ts @@ -11,6 +11,7 @@ import type { ReplayVarScope } from '../../replay/vars.ts'; import type { SnapshotState } from '../../kernel/snapshot.ts'; import { emitDiagnostic } from '../../utils/diagnostics.ts'; import { sleep } from '../../utils/timeouts.ts'; +import { invokeMaestroClickPoint } from './runtime-click.ts'; import { pointForMaestroTapOnTarget, swipeCoordinatesFromTarget } from './runtime-geometry.ts'; import { dismissAndroidMaestroBlockingOverlay, @@ -149,15 +150,7 @@ export async function invokeMaestroTapPointPercent(params: { } const point = pointFromPercent(frame, xPercent, yPercent); - const response = await params.invoke({ - ...params.baseReq, - command: 'click', - positionals: [String(point.x), String(point.y)], - flags: { - ...params.baseReq.flags, - postGestureStabilization: true, - }, - }); + const response = await invokeMaestroClickPoint({ ...params, point }); if (response.ok) clearMaestroRecoverableInteraction(params.scope); return response; } @@ -566,15 +559,7 @@ async function clickMaestroResolvedTarget( point, }, }); - const response = await params.invoke({ - ...params.baseReq, - command: 'click', - positionals: [String(point.x), String(point.y)], - flags: { - ...params.baseReq.flags, - postGestureStabilization: true, - }, - }); + const response = await invokeMaestroClickPoint({ ...params, point }); if (response.ok) { clearMaestroVisibleContext(params.scope); rememberMaestroRecoverableInteraction(params.scope, { diff --git a/src/platforms/android/input-ownership.ts b/src/core/android-input-ownership.ts similarity index 100% rename from src/platforms/android/input-ownership.ts rename to src/core/android-input-ownership.ts diff --git a/src/daemon/handlers/snapshot-capture.ts b/src/daemon/handlers/snapshot-capture.ts index 346ad0b613..8cccd61444 100644 --- a/src/daemon/handlers/snapshot-capture.ts +++ b/src/daemon/handlers/snapshot-capture.ts @@ -16,7 +16,7 @@ import { import { annotateCoveredSnapshotNodes } from '../../snapshot/snapshot-occlusion.ts'; import { normalizeSnapshotTree } from '../../snapshot/snapshot-tree.ts'; export { buildSnapshotVisibility } from '../../snapshot/snapshot-visibility.ts'; -import { isAndroidInputMethodSnapshotNode } from '../../platforms/android/input-method-overlays.ts'; +import { isAndroidInputMethodSnapshotNode } from '../../snapshot/android-input-method-overlays.ts'; import type { SessionState } from '../types.ts'; import { ANDROID_FRESHNESS_RETRY_DEADLINE_MS, diff --git a/src/platforms/android/__tests__/input-ownership.test.ts b/src/platforms/android/__tests__/input-ownership.test.ts index 814f827051..f9ffacddfb 100644 --- a/src/platforms/android/__tests__/input-ownership.test.ts +++ b/src/platforms/android/__tests__/input-ownership.test.ts @@ -4,7 +4,7 @@ import { classifyAndroidInputOwnership, parseAndroidInputMethodPackage, readAndroidActiveInputMethodPackage, -} from '../input-ownership.ts'; +} from '../../../core/android-input-ownership.ts'; test('classifies active input method package as IME-owned', () => { assert.deepEqual( diff --git a/src/platforms/android/device-input-state.ts b/src/platforms/android/device-input-state.ts index 5702be031c..8ab93982d7 100644 --- a/src/platforms/android/device-input-state.ts +++ b/src/platforms/android/device-input-state.ts @@ -13,7 +13,7 @@ import { isFallbackAndroidInputMethodResource, readAndroidActiveInputMethodPackage, type AndroidInputOwner, -} from './input-ownership.ts'; +} from '../../core/android-input-ownership.ts'; const ANDROID_INPUT_TYPE_CLASS_MASK = 0x0000000f; const ANDROID_INPUT_TYPE_CLASS_TEXT = 0x00000001; diff --git a/src/platforms/android/fill-verification.ts b/src/platforms/android/fill-verification.ts index f4461c4d3d..7a165e434d 100644 --- a/src/platforms/android/fill-verification.ts +++ b/src/platforms/android/fill-verification.ts @@ -10,7 +10,7 @@ import { } from '../fill-diagnostics.ts'; import { sleep } from './adb.ts'; import { getAndroidKeyboardState } from './device-input-state.ts'; -import { isAndroidInputMethodOwnedNode } from './input-ownership.ts'; +import { isAndroidInputMethodOwnedNode } from '../../core/android-input-ownership.ts'; import { captureAndroidUiHierarchyXml } from './snapshot.ts'; import { androidUiNodes, type AndroidUiNodeMetadata } from './ui-hierarchy.ts'; diff --git a/src/platforms/android/snapshot-content-recovery.ts b/src/platforms/android/snapshot-content-recovery.ts index 9199b197dc..cdc81cd976 100644 --- a/src/platforms/android/snapshot-content-recovery.ts +++ b/src/platforms/android/snapshot-content-recovery.ts @@ -1,5 +1,5 @@ import type { AndroidSnapshotBackendMetadata } from './snapshot-types.ts'; -import { isAndroidInputMethodOwnedNode } from './input-ownership.ts'; +import { isAndroidInputMethodOwnedNode } from '../../core/android-input-ownership.ts'; import { androidUiNodes, type AndroidUiNodeMetadata } from './ui-hierarchy.ts'; const ANDROID_WINDOW_TYPE_APPLICATION = 1; @@ -7,6 +7,8 @@ const MAX_REPORTED_WINDOW_TYPES = 8; const MIN_FOREGROUND_APP_MEANINGFUL_NODES = 2; const MIN_INPUT_METHOD_MEANINGFUL_NODES = 2; const ANDROID_SYSTEM_PACKAGES = new Set(['android', 'com.android.systemui']); +const INSUFFICIENT_APP_CONTENT_REASON = + 'Android snapshot helper returned insufficient application window content'; export type AndroidHelperContentRecoveryDecision = { reason: 'empty-helper-output' | 'system-window-only' | 'content-poor-app-window'; @@ -35,7 +37,7 @@ export function classifyAndroidHelperContentRecovery( if (metadata.backend !== 'android-helper') return undefined; const summary = summarizeAndroidHelperXml(xml, options.foregroundAppPackage); - if (summary.nodeCount === 0 || metadata.nodeCount === 0 || metadata.rootPresent === false) { + if (isEmptyHelperOutput(summary, metadata)) { return buildRecoveryDecision( summary, metadata, @@ -44,16 +46,8 @@ export function classifyAndroidHelperContentRecovery( ); } - const foregroundAppMeaningfulNodeCount = summary.foregroundAppMeaningfulNodeCount; - if ( - foregroundAppMeaningfulNodeCount !== undefined && - (foregroundAppMeaningfulNodeCount === 0 || - (foregroundAppMeaningfulNodeCount < MIN_FOREGROUND_APP_MEANINGFUL_NODES && - summary.meaningfulNodeCount > foregroundAppMeaningfulNodeCount)) - ) { - if (summary.inputMethodMeaningfulNodeCount >= MIN_INPUT_METHOD_MEANINGFUL_NODES) { - return undefined; - } + if (isForegroundAppContentHiddenByInputMethod(summary)) return undefined; + if (isForegroundAppContentPoor(summary)) { return buildRecoveryDecision( summary, metadata, @@ -62,34 +56,25 @@ export function classifyAndroidHelperContentRecovery( ); } - if ( - foregroundAppMeaningfulNodeCount === undefined && - summary.applicationWindowRootCount > 0 && - summary.applicationMeaningfulNodeCount < MIN_FOREGROUND_APP_MEANINGFUL_NODES - ) { + if (isApplicationWindowContentPoor(summary)) { return buildRecoveryDecision( summary, metadata, 'content-poor-app-window', - 'Android snapshot helper returned insufficient application window content', + INSUFFICIENT_APP_CONTENT_REASON, ); } - if ( - foregroundAppMeaningfulNodeCount === undefined && - summary.windowRootCount === 0 && - (metadata.windowCount ?? 0) > 1 && - summary.nonSystemMeaningfulNodeCount < MIN_FOREGROUND_APP_MEANINGFUL_NODES - ) { + if (isWindowlessMultiWindowContentPoor(summary, metadata)) { return buildRecoveryDecision( summary, metadata, 'content-poor-app-window', - 'Android snapshot helper returned insufficient application window content', + INSUFFICIENT_APP_CONTENT_REASON, ); } - if (summary.windowRootCount > 0 && summary.applicationWindowRootCount === 0) { + if (isSystemWindowOnly(summary)) { return buildRecoveryDecision( summary, metadata, @@ -101,6 +86,54 @@ export function classifyAndroidHelperContentRecovery( return undefined; } +function isEmptyHelperOutput( + summary: AndroidHelperXmlSummary, + metadata: AndroidSnapshotBackendMetadata, +): boolean { + return summary.nodeCount === 0 || metadata.nodeCount === 0 || metadata.rootPresent === false; +} + +function isForegroundAppContentHiddenByInputMethod(summary: AndroidHelperXmlSummary): boolean { + return ( + isForegroundAppContentPoor(summary) && + summary.inputMethodMeaningfulNodeCount >= MIN_INPUT_METHOD_MEANINGFUL_NODES + ); +} + +function isForegroundAppContentPoor(summary: AndroidHelperXmlSummary): boolean { + const foregroundCount = summary.foregroundAppMeaningfulNodeCount; + if (foregroundCount === undefined) return false; + if (foregroundCount === 0) return true; + return ( + foregroundCount < MIN_FOREGROUND_APP_MEANINGFUL_NODES && + summary.meaningfulNodeCount > foregroundCount + ); +} + +function isApplicationWindowContentPoor(summary: AndroidHelperXmlSummary): boolean { + return ( + summary.foregroundAppMeaningfulNodeCount === undefined && + summary.applicationWindowRootCount > 0 && + summary.applicationMeaningfulNodeCount < MIN_FOREGROUND_APP_MEANINGFUL_NODES + ); +} + +function isWindowlessMultiWindowContentPoor( + summary: AndroidHelperXmlSummary, + metadata: AndroidSnapshotBackendMetadata, +): boolean { + return ( + summary.foregroundAppMeaningfulNodeCount === undefined && + summary.windowRootCount === 0 && + (metadata.windowCount ?? 0) > 1 && + summary.nonSystemMeaningfulNodeCount < MIN_FOREGROUND_APP_MEANINGFUL_NODES + ); +} + +function isSystemWindowOnly(summary: AndroidHelperXmlSummary): boolean { + return summary.windowRootCount > 0 && summary.applicationWindowRootCount === 0; +} + function buildRecoveryDecision( summary: AndroidHelperXmlSummary, metadata: AndroidSnapshotBackendMetadata, diff --git a/src/platforms/android/input-method-overlays.ts b/src/snapshot/android-input-method-overlays.ts similarity index 94% rename from src/platforms/android/input-method-overlays.ts rename to src/snapshot/android-input-method-overlays.ts index a482fb226a..14945e632a 100644 --- a/src/platforms/android/input-method-overlays.ts +++ b/src/snapshot/android-input-method-overlays.ts @@ -1,5 +1,5 @@ -import type { RawSnapshotNode, SnapshotNode, SnapshotState } from '../../kernel/snapshot.ts'; -import { classifyAndroidInputOwnership } from './input-ownership.ts'; +import { classifyAndroidInputOwnership } from '../core/android-input-ownership.ts'; +import type { RawSnapshotNode, SnapshotNode, SnapshotState } from '../kernel/snapshot.ts'; const GBOARD_PACKAGE = 'com.google.android.inputmethod.latin'; const GBOARD_HANDWRITING_TITLE = 'Try out your stylus'; diff --git a/src/snapshot/snapshot-occlusion.ts b/src/snapshot/snapshot-occlusion.ts index 3412ab8d2f..522907eceb 100644 --- a/src/snapshot/snapshot-occlusion.ts +++ b/src/snapshot/snapshot-occlusion.ts @@ -160,21 +160,34 @@ function isAdditionalOverlayRootNode( options: SnapshotOcclusionOptions, ): boolean { if (options.isAdditionalOverlayNode?.(node) !== true) return false; + return !hasRenderableAdditionalOverlayAncestor(node, byIndex, options); +} + +function hasRenderableAdditionalOverlayAncestor( + node: RawSnapshotNode, + byIndex: Map, + options: SnapshotOcclusionOptions, +): boolean { let current = typeof node.parentIndex === 'number' ? byIndex.get(node.parentIndex) : undefined; const visited = new Set(); while (current && !visited.has(current.index)) { - if ( - options.isAdditionalOverlayNode?.(current) === true && - positiveRect(current.rect) && - !isViewportRoot(current) - ) { - return false; - } + if (isRenderableAdditionalOverlayNode(current, options)) return true; visited.add(current.index); current = typeof current.parentIndex === 'number' ? byIndex.get(current.parentIndex) : undefined; } - return true; + return false; +} + +function isRenderableAdditionalOverlayNode( + node: RawSnapshotNode, + options: SnapshotOcclusionOptions, +): boolean { + return ( + options.isAdditionalOverlayNode?.(node) === true && + positiveRect(node.rect) !== null && + !isViewportRoot(node) + ); } function isSemanticTouchNode(node: RawSnapshotNode): boolean { From 92a48da845021a7a768259764622506e4b4c51fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 8 Jul 2026 17:43:30 +0200 Subject: [PATCH 3/4] fix: detect localized Gboard tutorial overlays --- .../android-input-method-overlays.test.ts | 57 +++++++++++++++++++ src/snapshot/android-input-method-overlays.ts | 22 ++++++- 2 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 src/snapshot/__tests__/android-input-method-overlays.test.ts diff --git a/src/snapshot/__tests__/android-input-method-overlays.test.ts b/src/snapshot/__tests__/android-input-method-overlays.test.ts new file mode 100644 index 0000000000..eb7cfc8c70 --- /dev/null +++ b/src/snapshot/__tests__/android-input-method-overlays.test.ts @@ -0,0 +1,57 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { attachRefs, type RawSnapshotNode, type SnapshotState } from '../../kernel/snapshot.ts'; +import { + findAndroidGboardHandwritingTutorialCancel, + hasAndroidGboardHandwritingTutorial, +} from '../android-input-method-overlays.ts'; + +test('detects localized Gboard handwriting tutorial by close button id', () => { + const snapshot = snapshotState([ + gboardNode(0, { + label: 'Prueba tu stylus', + value: 'Prueba tu stylus', + }), + gboardNode(1, { + identifier: 'android:id/closeButton', + label: 'Cancelar', + value: 'Cancelar', + }), + ]); + + assert.equal(hasAndroidGboardHandwritingTutorial(snapshot), true); + assert.equal(findAndroidGboardHandwritingTutorialCancel(snapshot)?.index, 1); +}); + +test('ignores non-Gboard close buttons', () => { + const snapshot = snapshotState([ + { + index: 0, + type: 'Button', + bundleId: 'com.example.app', + identifier: 'android:id/closeButton', + label: 'Cancel', + }, + ]); + + assert.equal(hasAndroidGboardHandwritingTutorial(snapshot), false); + assert.equal(findAndroidGboardHandwritingTutorialCancel(snapshot), undefined); +}); + +function snapshotState(nodes: RawSnapshotNode[]): SnapshotState { + return { + nodes: attachRefs(nodes), + createdAt: Date.now(), + backend: 'android', + presentationKey: 'test', + }; +} + +function gboardNode(index: number, node: Partial): RawSnapshotNode { + return { + index, + type: 'TextView', + bundleId: 'com.google.android.inputmethod.latin', + ...node, + }; +} diff --git a/src/snapshot/android-input-method-overlays.ts b/src/snapshot/android-input-method-overlays.ts index 14945e632a..91f470f73a 100644 --- a/src/snapshot/android-input-method-overlays.ts +++ b/src/snapshot/android-input-method-overlays.ts @@ -19,12 +19,17 @@ export function isAndroidInputMethodSnapshotNode( export function findAndroidGboardHandwritingTutorialCancel( snapshot: SnapshotState, ): SnapshotNode | undefined { - if (!snapshot.nodes.some(isAndroidGboardHandwritingTutorialTitle)) return undefined; - return snapshot.nodes.find(isAndroidGboardHandwritingTutorialCancel); + return ( + snapshot.nodes.find(isAndroidGboardHandwritingTutorialCloseButton) ?? + findEnglishAndroidGboardHandwritingTutorialCancel(snapshot) + ); } export function hasAndroidGboardHandwritingTutorial(snapshot: SnapshotState): boolean { - return snapshot.nodes.some(isAndroidGboardHandwritingTutorialTitle); + return ( + snapshot.nodes.some(isAndroidGboardHandwritingTutorialCloseButton) || + snapshot.nodes.some(isAndroidGboardHandwritingTutorialTitle) + ); } function isAndroidGboardHandwritingTutorialTitle(node: SnapshotNode): boolean { @@ -37,6 +42,17 @@ function isAndroidGboardHandwritingTutorialCancel(node: SnapshotNode): boolean { return nodeTextValues(node).includes(GBOARD_HANDWRITING_CANCEL_LABEL); } +function isAndroidGboardHandwritingTutorialCloseButton(node: SnapshotNode): boolean { + return isGboardSnapshotNode(node) && node.identifier === GBOARD_HANDWRITING_CANCEL_ID; +} + +function findEnglishAndroidGboardHandwritingTutorialCancel( + snapshot: SnapshotState, +): SnapshotNode | undefined { + if (!snapshot.nodes.some(isAndroidGboardHandwritingTutorialTitle)) return undefined; + return snapshot.nodes.find(isAndroidGboardHandwritingTutorialCancel); +} + function isGboardSnapshotNode(node: SnapshotNode): boolean { return node.bundleId === GBOARD_PACKAGE && isAndroidInputMethodSnapshotNode(node); } From 6799fa71992848bfe6d1dde4b893a4a56981a883 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Wed, 8 Jul 2026 18:01:07 +0200 Subject: [PATCH 4/4] fix: keep Android IME overlay handling passive --- .../__tests__/runtime-assertions.test.ts | 61 +--------------- .../__tests__/runtime-interactions.test.ts | 70 ------------------- .../maestro/runtime-android-overlays.ts | 48 ------------- src/compat/maestro/runtime-assertions.ts | 43 ------------ src/compat/maestro/runtime-click.ts | 9 +-- src/compat/maestro/runtime-interactions.ts | 49 +------------ .../android-input-method-overlays.test.ts | 57 --------------- src/snapshot/android-input-method-overlays.ts | 52 +------------- 8 files changed, 5 insertions(+), 384 deletions(-) delete mode 100644 src/compat/maestro/runtime-android-overlays.ts delete mode 100644 src/snapshot/__tests__/android-input-method-overlays.test.ts diff --git a/src/compat/maestro/__tests__/runtime-assertions.test.ts b/src/compat/maestro/__tests__/runtime-assertions.test.ts index 5d53fac5e6..ded03fa15e 100644 --- a/src/compat/maestro/__tests__/runtime-assertions.test.ts +++ b/src/compat/maestro/__tests__/runtime-assertions.test.ts @@ -555,38 +555,6 @@ test('invokeMaestroAssertVisible does not use Android raw fallback for generated ); }); -test('invokeMaestroAssertVisible dismisses Android Gboard handwriting tutorial before native wait', async () => { - const calls: Array<[string, string[] | undefined]> = []; - const snapshots = [ - gboardHandwritingTutorialSnapshot(), - snapshot([node('Input', { type: 'android.view.View' })]), - ]; - const response = await invokeMaestroAssertVisible({ - baseReq: { - token: 't', - session: 's', - flags: { platform: 'android' }, - }, - positionals: ['label="Input" || text="Input" || id="Input"', '60000'], - invoke: async (req): Promise => { - calls.push([req.command, req.positionals]); - if (req.command === 'snapshot') { - return { ok: true, data: snapshots.shift() ?? snapshot([node('Input')]) }; - } - if (req.command === 'click') return { ok: true, data: {} }; - if (req.command === 'wait') return { ok: true, data: { matches: 1 } }; - return { ok: false, error: { code: 'UNEXPECTED_COMMAND', message: req.command } }; - }, - }); - - assert.equal(response.ok, true); - assert.deepEqual(calls, [ - ['snapshot', []], - ['click', ['588', '2758']], - ['snapshot', []], - ]); -}); - test('invokeMaestroAssertVisible bounds Android verification retries after native wait succeeds', async () => { vi.useFakeTimers(); @@ -613,9 +581,9 @@ test('invokeMaestroAssertVisible bounds Android verification retries after nativ assert.equal(response.ok, false); assert.deepEqual(calls.slice(0, 3), [ - ['snapshot', []], ['wait', ['Input', '60000']], ['snapshot', []], + ['snapshot', []], ]); assert.ok(calls.filter(([command]) => command === 'snapshot').length < 40); }); @@ -883,33 +851,6 @@ function node( }; } -function gboardHandwritingTutorialSnapshot(): SnapshotState { - return snapshot([ - node('Push Article', { - index: 2, - ref: 'e2', - type: 'android.widget.Button', - bundleId: 'org.reactnavigation.playground', - }), - node('Try out your stylus', { - index: 3, - ref: 'e3', - value: 'Try out your stylus', - bundleId: 'com.google.android.inputmethod.latin', - rect: { x: 376, y: 1650, width: 592, height: 90 }, - }), - node('Cancel', { - index: 4, - ref: 'e4', - value: 'Cancel', - identifier: 'android:id/closeButton', - type: 'android.widget.Button', - bundleId: 'com.google.android.inputmethod.latin', - rect: { x: 450, y: 2699, width: 276, height: 118 }, - }), - ]); -} - test('invokeMaestroAssertNotVisible accepts timeout overrides for short extended waits', async () => { vi.spyOn(Date, 'now').mockReturnValueOnce(0).mockReturnValueOnce(0).mockReturnValueOnce(300); diff --git a/src/compat/maestro/__tests__/runtime-interactions.test.ts b/src/compat/maestro/__tests__/runtime-interactions.test.ts index a9dcf86fb2..db499c7d21 100644 --- a/src/compat/maestro/__tests__/runtime-interactions.test.ts +++ b/src/compat/maestro/__tests__/runtime-interactions.test.ts @@ -211,35 +211,6 @@ test('invokeMaestroTapOn resolves visible Android non-interactive text from a re expect(clicks).toEqual([['248', '231']]); }); -test('invokeMaestroTapOn dismisses Android Gboard handwriting tutorial before app tap', async () => { - const snapshots = [gboardHandwritingTutorialSnapshot(), buttonSnapshot('Push Article')]; - const clicks: string[][] = []; - const response = await invokeMaestroTapOn({ - baseReq: { - token: 'test', - session: 'android-input', - flags: { platform: 'android' }, - }, - positionals: ['label="Push Article" || text="Push Article" || id="Push Article"'], - invoke: async (req: DaemonRequest): Promise => { - if (req.command === 'snapshot') { - return { ok: true, data: snapshots.shift() ?? buttonSnapshot('Push Article') }; - } - if (req.command === 'click') { - clicks.push(req.positionals ?? []); - return { ok: true, data: {} }; - } - return { ok: false, error: { code: 'UNEXPECTED_COMMAND', message: req.command } }; - }, - }); - - expect(response.ok).toBe(true); - expect(clicks).toEqual([ - ['588', '2758'], - ['201', '149'], - ]); -}); - test('invokeMaestroTapOn taps resolved iOS buttons by coordinates', async () => { const { response, clicks } = await runTapOn( 'label="Pop to top" || text="Pop to top" || id="Pop to top"', @@ -720,47 +691,6 @@ function overlayDismissButtonSnapshot(): SnapshotState { }; } -function gboardHandwritingTutorialSnapshot(): SnapshotState { - return { - createdAt: Date.now(), - nodes: [ - appNode(), - windowNode(), - { - index: 2, - ref: 'e3', - type: 'android.widget.Button', - label: 'Push Article', - bundleId: 'org.reactnavigation.playground', - depth: 21, - parentIndex: 1, - rect: { x: 142, y: 128.66666412353516, width: 118, height: 40 }, - }, - { - index: 3, - ref: 'e4', - type: 'android.widget.TextView', - label: 'Try out your stylus', - value: 'Try out your stylus', - bundleId: 'com.google.android.inputmethod.latin', - depth: 3, - rect: { x: 376, y: 1650, width: 592, height: 90 }, - }, - { - index: 4, - ref: 'e5', - type: 'android.widget.Button', - label: 'Cancel', - value: 'Cancel', - identifier: 'android:id/closeButton', - bundleId: 'com.google.android.inputmethod.latin', - depth: 4, - rect: { x: 450, y: 2699, width: 276, height: 118 }, - }, - ], - }; -} - function truncatedContentSnapshot(): SnapshotState { return { createdAt: Date.now(), diff --git a/src/compat/maestro/runtime-android-overlays.ts b/src/compat/maestro/runtime-android-overlays.ts deleted file mode 100644 index 67c390aea8..0000000000 --- a/src/compat/maestro/runtime-android-overlays.ts +++ /dev/null @@ -1,48 +0,0 @@ -import type { SnapshotNode, SnapshotState } from '../../kernel/snapshot.ts'; -import { - findAndroidGboardHandwritingTutorialCancel, - hasAndroidGboardHandwritingTutorial, - isAndroidInputMethodSnapshotNode, -} from '../../snapshot/android-input-method-overlays.ts'; -import { emitDiagnostic } from '../../utils/diagnostics.ts'; -import { centerSnapshotRectPoint, invokeMaestroClickPoint } from './runtime-click.ts'; -import type { MaestroRuntimeInvoke, ReplayBaseRequest } from './runtime-support.ts'; - -export async function dismissAndroidMaestroBlockingOverlay(params: { - baseReq: ReplayBaseRequest; - invoke: MaestroRuntimeInvoke; - snapshot: SnapshotState; - targetNode?: SnapshotNode; - selector: string; -}): Promise { - if (params.baseReq.flags?.platform !== 'android') return false; - if (isAndroidInputMethodSnapshotNode(params.targetNode)) return false; - - const cancel = findAndroidGboardHandwritingTutorialCancel(params.snapshot); - if (!cancel?.rect) return false; - - const point = centerSnapshotRectPoint(cancel.rect); - emitDiagnostic({ - level: 'info', - phase: 'maestro_android_blocking_overlay_dismiss', - data: { - selector: params.selector, - overlay: 'gboard-handwriting-tutorial', - nodeIndex: cancel.index, - point, - }, - }); - - const response = await invokeMaestroClickPoint({ ...params, point }); - return response.ok; -} - -export function hasAndroidMaestroBlockingOverlay(params: { - baseReq: ReplayBaseRequest; - snapshot: SnapshotState; - targetNode?: SnapshotNode; -}): boolean { - if (params.baseReq.flags?.platform !== 'android') return false; - if (isAndroidInputMethodSnapshotNode(params.targetNode)) return false; - return hasAndroidGboardHandwritingTutorial(params.snapshot); -} diff --git a/src/compat/maestro/runtime-assertions.ts b/src/compat/maestro/runtime-assertions.ts index e524e83899..700b9797d8 100644 --- a/src/compat/maestro/runtime-assertions.ts +++ b/src/compat/maestro/runtime-assertions.ts @@ -8,7 +8,6 @@ import { emitDiagnostic } from '../../utils/diagnostics.ts'; import type { Point, SnapshotState } from '../../kernel/snapshot.ts'; import { buildSnapshotDisplayLines } from '../../snapshot/snapshot-lines.ts'; import { sleep } from '../../utils/timeouts.ts'; -import { dismissAndroidMaestroBlockingOverlay } from './runtime-android-overlays.ts'; import { pointForMaestroTapOnTarget } from './runtime-geometry.ts'; import { captureMaestroSnapshot, @@ -88,13 +87,6 @@ async function invokeNativeMaestroVisibleWaitWithSnapshotFallback( nativeWaitQuery: string, ): Promise { const nativeStartedAt = Date.now(); - const preflightResponse = await maybeDismissAndroidBlockingOverlayBeforeVisibleWait( - params, - args, - nativeStartedAt, - ); - if (preflightResponse) return preflightResponse; - const nativeResponse = await runNativeVisibleWait(params, args, nativeWaitQuery); if (nativeResponse.ok) { if (shouldVerifyNativeVisibleWait(params.baseReq)) { @@ -134,45 +126,10 @@ async function invokeNativeMaestroVisibleWaitWithSnapshotFallback( ); } -async function maybeDismissAndroidBlockingOverlayBeforeVisibleWait( - params: MaestroAssertionRuntimeParams, - args: MaestroVisibilityAssertionArgs, - startedAt: number, -): Promise { - if (!shouldPreflightAndroidBlockingOverlay(params.baseReq, args)) return null; - - const sample = await readMaestroVisibilitySample(params, args.selector, 'assertVisible'); - if (sample.visible) return visibleAssertionResponse(sample.response, args.selector, startedAt); - if (!sample.snapshot) return null; - - const dismissed = await dismissAndroidMaestroBlockingOverlay({ - baseReq: params.baseReq, - invoke: params.invoke, - snapshot: sample.snapshot, - selector: args.selector, - }); - if (!dismissed) return null; - - return await invokeSnapshotMaestroAssertVisible(params, { - ...args, - timeoutMs: visibleAssertionRetryTimeoutMs(args.timeoutMs), - }); -} - function shouldVerifyNativeVisibleWait(baseReq: ReplayBaseRequest): boolean { return baseReq.flags?.platform === 'android'; } -function shouldPreflightAndroidBlockingOverlay( - baseReq: ReplayBaseRequest, - args: MaestroVisibilityAssertionArgs, -): boolean { - // Long launch waits can otherwise spend the full native wait budget before - // snapshot verification notices environment-owned keyboard overlays. Ordinary - // waits rely on the normal Android verification snapshot and core occlusion. - return baseReq.flags?.platform === 'android' && args.timeoutMs >= 30_000; -} - async function runNativeVisibleWait( params: MaestroAssertionRuntimeParams, args: MaestroVisibilityAssertionArgs, diff --git a/src/compat/maestro/runtime-click.ts b/src/compat/maestro/runtime-click.ts index 6eb58b1765..fced71eb29 100644 --- a/src/compat/maestro/runtime-click.ts +++ b/src/compat/maestro/runtime-click.ts @@ -1,5 +1,5 @@ import type { DaemonResponse } from '../../daemon/types.ts'; -import type { Point, SnapshotNode } from '../../kernel/snapshot.ts'; +import type { Point } from '../../kernel/snapshot.ts'; import type { MaestroRuntimeInvoke, ReplayBaseRequest } from './runtime-support.ts'; export async function invokeMaestroClickPoint(params: { @@ -17,10 +17,3 @@ export async function invokeMaestroClickPoint(params: { }, }); } - -export function centerSnapshotRectPoint(rect: NonNullable): Point { - return { - x: Math.round(rect.x + rect.width / 2), - y: Math.round(rect.y + rect.height / 2), - }; -} diff --git a/src/compat/maestro/runtime-interactions.ts b/src/compat/maestro/runtime-interactions.ts index ebcb536251..1c73e16cde 100644 --- a/src/compat/maestro/runtime-interactions.ts +++ b/src/compat/maestro/runtime-interactions.ts @@ -13,10 +13,6 @@ import { emitDiagnostic } from '../../utils/diagnostics.ts'; import { sleep } from '../../utils/timeouts.ts'; import { invokeMaestroClickPoint } from './runtime-click.ts'; import { pointForMaestroTapOnTarget, swipeCoordinatesFromTarget } from './runtime-geometry.ts'; -import { - dismissAndroidMaestroBlockingOverlay, - hasAndroidMaestroBlockingOverlay, -} from './runtime-android-overlays.ts'; import { captureMaestroSnapshot, clearMaestroRecoverableInteraction, @@ -61,10 +57,6 @@ type MaestroTapOnParams = { scope?: ReplayVarScope; }; -type MaestroTapOnAttemptState = { - dismissedBlockingOverlay: boolean; -}; - type MaestroScreenSwipeResolution = | { ok: true; @@ -194,9 +186,8 @@ export async function invokeMaestroTapOn(params: MaestroTapOnParams): Promise { const fuzzyTextQuery = extractMaestroVisibleTextQuery(selector); - const attempt = await invokeMaestroResolvedTapOn(params, selector, options, attemptState); + const attempt = await invokeMaestroResolvedTapOn(params, selector, options); if (attempt.response.ok) return { retry: false, response: attempt.response }; if (attempt.targetResolved && fuzzyTextQuery) { return await invokeMaestroFuzzyTapOn(params, fuzzyTextQuery); @@ -492,46 +482,11 @@ async function invokeMaestroResolvedTapOn( params: MaestroTapOnParams, selector: string, options: MaestroTapOnOptions, - attemptState: MaestroTapOnAttemptState, ): Promise<{ response: DaemonResponse; targetResolved: boolean }> { const target = await resolveMaestroInteractionTarget(params, selector, options, 'tapOn', { promoteTapTarget: true, }); if (!target.ok) return { response: target.response, targetResolved: false }; - if ( - attemptState.dismissedBlockingOverlay && - hasAndroidMaestroBlockingOverlay({ - baseReq: params.baseReq, - snapshot: target.snapshot, - targetNode: target.target.node, - }) - ) { - return { - response: errorResponse( - 'COMMAND_FAILED', - 'Android input overlay is still blocking Maestro tap after dismissal attempt.', - { selector }, - ), - targetResolved: false, - }; - } - const dismissedOverlay = await dismissAndroidMaestroBlockingOverlay({ - baseReq: params.baseReq, - invoke: params.invoke, - snapshot: target.snapshot, - targetNode: target.target.node, - selector, - }); - if (dismissedOverlay) { - attemptState.dismissedBlockingOverlay = true; - return { - response: errorResponse( - 'COMMAND_FAILED', - 'Dismissed Android input overlay before Maestro tap; retrying target resolution.', - ), - targetResolved: false, - }; - } return await clickMaestroResolvedTarget(params, selector, target.target, options); } diff --git a/src/snapshot/__tests__/android-input-method-overlays.test.ts b/src/snapshot/__tests__/android-input-method-overlays.test.ts deleted file mode 100644 index eb7cfc8c70..0000000000 --- a/src/snapshot/__tests__/android-input-method-overlays.test.ts +++ /dev/null @@ -1,57 +0,0 @@ -import assert from 'node:assert/strict'; -import { test } from 'vitest'; -import { attachRefs, type RawSnapshotNode, type SnapshotState } from '../../kernel/snapshot.ts'; -import { - findAndroidGboardHandwritingTutorialCancel, - hasAndroidGboardHandwritingTutorial, -} from '../android-input-method-overlays.ts'; - -test('detects localized Gboard handwriting tutorial by close button id', () => { - const snapshot = snapshotState([ - gboardNode(0, { - label: 'Prueba tu stylus', - value: 'Prueba tu stylus', - }), - gboardNode(1, { - identifier: 'android:id/closeButton', - label: 'Cancelar', - value: 'Cancelar', - }), - ]); - - assert.equal(hasAndroidGboardHandwritingTutorial(snapshot), true); - assert.equal(findAndroidGboardHandwritingTutorialCancel(snapshot)?.index, 1); -}); - -test('ignores non-Gboard close buttons', () => { - const snapshot = snapshotState([ - { - index: 0, - type: 'Button', - bundleId: 'com.example.app', - identifier: 'android:id/closeButton', - label: 'Cancel', - }, - ]); - - assert.equal(hasAndroidGboardHandwritingTutorial(snapshot), false); - assert.equal(findAndroidGboardHandwritingTutorialCancel(snapshot), undefined); -}); - -function snapshotState(nodes: RawSnapshotNode[]): SnapshotState { - return { - nodes: attachRefs(nodes), - createdAt: Date.now(), - backend: 'android', - presentationKey: 'test', - }; -} - -function gboardNode(index: number, node: Partial): RawSnapshotNode { - return { - index, - type: 'TextView', - bundleId: 'com.google.android.inputmethod.latin', - ...node, - }; -} diff --git a/src/snapshot/android-input-method-overlays.ts b/src/snapshot/android-input-method-overlays.ts index 91f470f73a..2bd29a3bf9 100644 --- a/src/snapshot/android-input-method-overlays.ts +++ b/src/snapshot/android-input-method-overlays.ts @@ -1,10 +1,5 @@ import { classifyAndroidInputOwnership } from '../core/android-input-ownership.ts'; -import type { RawSnapshotNode, SnapshotNode, SnapshotState } from '../kernel/snapshot.ts'; - -const GBOARD_PACKAGE = 'com.google.android.inputmethod.latin'; -const GBOARD_HANDWRITING_TITLE = 'Try out your stylus'; -const GBOARD_HANDWRITING_CANCEL_ID = 'android:id/closeButton'; -const GBOARD_HANDWRITING_CANCEL_LABEL = 'Cancel'; +import type { RawSnapshotNode } from '../kernel/snapshot.ts'; export function isAndroidInputMethodSnapshotNode( node: Pick | undefined, @@ -15,48 +10,3 @@ export function isAndroidInputMethodSnapshotNode( resourceId: node.identifier, }).inputMethodOwned; } - -export function findAndroidGboardHandwritingTutorialCancel( - snapshot: SnapshotState, -): SnapshotNode | undefined { - return ( - snapshot.nodes.find(isAndroidGboardHandwritingTutorialCloseButton) ?? - findEnglishAndroidGboardHandwritingTutorialCancel(snapshot) - ); -} - -export function hasAndroidGboardHandwritingTutorial(snapshot: SnapshotState): boolean { - return ( - snapshot.nodes.some(isAndroidGboardHandwritingTutorialCloseButton) || - snapshot.nodes.some(isAndroidGboardHandwritingTutorialTitle) - ); -} - -function isAndroidGboardHandwritingTutorialTitle(node: SnapshotNode): boolean { - return isGboardSnapshotNode(node) && nodeTextValues(node).includes(GBOARD_HANDWRITING_TITLE); -} - -function isAndroidGboardHandwritingTutorialCancel(node: SnapshotNode): boolean { - if (!isGboardSnapshotNode(node)) return false; - if (node.identifier === GBOARD_HANDWRITING_CANCEL_ID) return true; - return nodeTextValues(node).includes(GBOARD_HANDWRITING_CANCEL_LABEL); -} - -function isAndroidGboardHandwritingTutorialCloseButton(node: SnapshotNode): boolean { - return isGboardSnapshotNode(node) && node.identifier === GBOARD_HANDWRITING_CANCEL_ID; -} - -function findEnglishAndroidGboardHandwritingTutorialCancel( - snapshot: SnapshotState, -): SnapshotNode | undefined { - if (!snapshot.nodes.some(isAndroidGboardHandwritingTutorialTitle)) return undefined; - return snapshot.nodes.find(isAndroidGboardHandwritingTutorialCancel); -} - -function isGboardSnapshotNode(node: SnapshotNode): boolean { - return node.bundleId === GBOARD_PACKAGE && isAndroidInputMethodSnapshotNode(node); -} - -function nodeTextValues(node: SnapshotNode): string[] { - return [node.label, node.value].filter((value): value is string => Boolean(value)); -}